#!/usr/bin/env python# This software code is made available "AS IS" without warranties of any# kind. You may copy, display, modify and redistribute the software# code either by itself or as incorporated into your code; provided that# you do not remove any proprietary notices. Your use of this software# code is at your own risk and you waive any claim against Amazon# Digital Services, Inc. or its affiliates with respect to your use of# this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its# affiliates.importbase64importhmacimporthttplibtry:fromhashlibimportsha1asshaexceptImportError:importshaimporttimeimporturllibimporturlparseimportxml.saxDEFAULT_HOST='s3.amazonaws.com'PORTS_BY_SECURITY={True:443,False:80}METADATA_PREFIX='x-amz-meta-'AMAZON_HEADER_PREFIX='x-amz-'# generates the aws canonical string for the given parametersdefcanonical_string(method,bucket="",key="",query_args={},headers={},expires=None):interesting_headers={}forheader_keyinheaders:lk=header_key.lower()iflkin['content-md5','content-type','date']orlk.startswith(AMAZON_HEADER_PREFIX):interesting_headers[lk]=headers[header_key].strip()# these keys get empty strings if they don't existifnotinteresting_headers.has_key('content-type'):interesting_headers['content-type']=''ifnotinteresting_headers.has_key('content-md5'):interesting_headers['content-md5']=''# just in case someone used this. it's not necessary in this lib.ifinteresting_headers.has_key('x-amz-date'):interesting_headers['date']=''# if you're using expires for query string auth, then it trumps date# (and x-amz-date)ifexpires:interesting_headers['date']=str(expires)sorted_header_keys=interesting_headers.keys()sorted_header_keys.sort()buf="%s\n"%methodforheader_keyinsorted_header_keys:ifheader_key.startswith(AMAZON_HEADER_PREFIX):buf+="%s:%s\n"%(header_key,interesting_headers[header_key])else:buf+="%s\n"%interesting_headers[header_key]# append the bucket if it existsifbucket!="":buf+="/%s"%bucket# add the key. even if it doesn't exist, add the slashbuf+="/%s"%urllib.quote_plus(key)# handle special query string argumentsifquery_args.has_key("acl"):buf+="?acl"elifquery_args.has_key("torrent"):buf+="?torrent"elifquery_args.has_key("logging"):buf+="?logging"elifquery_args.has_key("location"):buf+="?location"returnbuf# computes the base64'ed hmac-sha hash of the canonical string and the secret# access key, optionally urlencoding the resultdefencode(aws_secret_access_key,str,urlencode=False):b64_hmac=base64.encodestring(hmac.new(aws_secret_access_key,str,sha).digest()).strip()ifurlencode:returnurllib.quote_plus(b64_hmac)else:returnb64_hmacdefmerge_meta(headers,metadata):final_headers=headers.copy()forkinmetadata.keys():final_headers[METADATA_PREFIX+k]=metadata[k]returnfinal_headers# builds the query arg stringdefquery_args_hash_to_string(query_args):query_string=""pairs=[]fork,vinquery_args.items():piece=kifv!=None:piece+="=%s"%urllib.quote_plus(str(v))pairs.append(piece)return'&'.join(pairs)classCallingFormat:PATH=1SUBDOMAIN=2VANITY=3defbuild_url_base(protocol,server,port,bucket,calling_format):url_base='%s://'%protocolifbucket=='':url_base+=serverelifcalling_format==CallingFormat.SUBDOMAIN:url_base+="%s.%s"%(bucket,server)elifcalling_format==CallingFormat.VANITY:url_base+=bucketelse:url_base+=serverurl_base+=":%s"%portif(bucket!='')and(calling_format==CallingFormat.PATH):url_base+="/%s"%bucketreturnurl_basebuild_url_base=staticmethod(build_url_base)classLocation:DEFAULT=NoneEU='EU'classAWSAuthConnection:def__init__(self,aws_access_key_id,aws_secret_access_key,is_secure=True,server=DEFAULT_HOST,port=None,calling_format=CallingFormat.SUBDOMAIN):ifnotport:port=PORTS_BY_SECURITY[is_secure]self.aws_access_key_id=aws_access_key_idself.aws_secret_access_key=aws_secret_access_keyself.is_secure=is_secureself.server=serverself.port=portself.calling_format=calling_formatdefcreate_bucket(self,bucket,headers={}):returnResponse(self._make_request('PUT',bucket,'',{},headers))defcreate_located_bucket(self,bucket,location=Location.DEFAULT,headers={}):iflocation==Location.DEFAULT:body=""else:body="<CreateBucketConstraint><LocationConstraint>"+ \
location+ \
"</LocationConstraint></CreateBucketConstraint>"returnResponse(self._make_request('PUT',bucket,'',{},headers,body))defcheck_bucket_exists(self,bucket):returnself._make_request('HEAD',bucket,'',{},{})deflist_bucket(self,bucket,options={},headers={}):returnListBucketResponse(self._make_request('GET',bucket,'',options,headers))defdelete_bucket(self,bucket,headers={}):returnResponse(self._make_request('DELETE',bucket,'',{},headers))defput(self,bucket,key,object,headers={}):ifnotisinstance(object,S3Object):object=S3Object(object)returnResponse(self._make_request('PUT',bucket,key,{},headers,object.data,object.metadata))defget(self,bucket,key,headers={}):returnGetResponse(self._make_request('GET',bucket,key,{},headers))defdelete(self,bucket,key,headers={}):returnResponse(self._make_request('DELETE',bucket,key,{},headers))defget_bucket_logging(self,bucket,headers={}):returnGetResponse(self._make_request('GET',bucket,'',{'logging':None},headers))defput_bucket_logging(self,bucket,logging_xml_doc,headers={}):returnResponse(self._make_request('PUT',bucket,'',{'logging':None},headers,logging_xml_doc))defget_bucket_acl(self,bucket,headers={}):returnself.get_acl(bucket,'',headers)defget_acl(self,bucket,key,headers={}):returnGetResponse(self._make_request('GET',bucket,key,{'acl':None},headers))defput_bucket_acl(self,bucket,acl_xml_document,headers={}):returnself.put_acl(bucket,'',acl_xml_document,headers)defput_acl(self,bucket,key,acl_xml_document,headers={}):returnResponse(self._make_request('PUT',bucket,key,{'acl':None},headers,acl_xml_document))deflist_all_my_buckets(self,headers={}):returnListAllMyBucketsResponse(self._make_request('GET','','',{},headers))defget_bucket_location(self,bucket):returnLocationResponse(self._make_request('GET',bucket,'',{'location':None}))# end public methodsdef_make_request(self,method,bucket='',key='',query_args={},headers={},data='',metadata={}):server=''ifbucket=='':server=self.serverelifself.calling_format==CallingFormat.SUBDOMAIN:server="%s.%s"%(bucket,self.server)elifself.calling_format==CallingFormat.VANITY:server=bucketelse:server=self.serverpath=''if(bucket!='')and(self.calling_format==CallingFormat.PATH):path+="/%s"%bucket# add the slash after the bucket regardless# the key will be appended if it is non-emptypath+="/%s"%urllib.quote_plus(key)# build the path_argument string# add the ? in all cases since # signature and credentials follow path argsiflen(query_args):path+="?"+query_args_hash_to_string(query_args)is_secure=self.is_securehost="%s:%d"%(server,self.port)whileTrue:if(is_secure):connection=httplib.HTTPSConnection(host)else:connection=httplib.HTTPConnection(host)final_headers=merge_meta(headers,metadata);# add auth headerself._add_aws_auth_header(final_headers,method,bucket,key,query_args)connection.request(method,path,data,final_headers)resp=connection.getresponse()ifresp.status<300orresp.status>=400:returnresp# handle redirectlocation=resp.getheader('location')ifnotlocation:returnresp# (close connection)resp.read()scheme,host,path,params,query,fragment \
=urlparse.urlparse(location)ifscheme=="http":is_secure=Trueelifscheme=="https":is_secure=Falseelse:raiseIOError("Not http/https: "+location)ifquery:path+="?"+query# retry with redirectdef_add_aws_auth_header(self,headers,method,bucket,key,query_args):ifnotheaders.has_key('Date'):headers['Date']=time.strftime("%a, %d %b %Y %X GMT",time.gmtime())c_string=canonical_string(method,bucket,key,query_args,headers)headers['Authorization']= \
"AWS %s:%s"%(self.aws_access_key_id,encode(self.aws_secret_access_key,c_string))classQueryStringAuthGenerator:# by default, expire in 1 minuteDEFAULT_EXPIRES_IN=60def__init__(self,aws_access_key_id,aws_secret_access_key,is_secure=True,server=DEFAULT_HOST,port=None,calling_format=CallingFormat.SUBDOMAIN):ifnotport:port=PORTS_BY_SECURITY[is_secure]self.aws_access_key_id=aws_access_key_idself.aws_secret_access_key=aws_secret_access_keyif(is_secure):self.protocol='https'else:self.protocol='http'self.is_secure=is_secureself.server=serverself.port=portself.calling_format=calling_formatself.__expires_in=QueryStringAuthGenerator.DEFAULT_EXPIRES_INself.__expires=None# for backwards compatibility with older versionsself.server_name="%s:%s"%(self.server,self.port)defset_expires_in(self,expires_in):self.__expires_in=expires_inself.__expires=Nonedefset_expires(self,expires):self.__expires=expiresself.__expires_in=Nonedefcreate_bucket(self,bucket,headers={}):returnself.generate_url('PUT',bucket,'',{},headers)deflist_bucket(self,bucket,options={},headers={}):returnself.generate_url('GET',bucket,'',options,headers)defdelete_bucket(self,bucket,headers={}):returnself.generate_url('DELETE',bucket,'',{},headers)defput(self,bucket,key,object,headers={}):ifnotisinstance(object,S3Object):object=S3Object(object)returnself.generate_url('PUT',bucket,key,{},merge_meta(headers,object.metadata))defget(self,bucket,key,headers={}):returnself.generate_url('GET',bucket,key,{},headers)defdelete(self,bucket,key,headers={}):returnself.generate_url('DELETE',bucket,key,{},headers)defget_bucket_logging(self,bucket,headers={}):returnself.generate_url('GET',bucket,'',{'logging':None},headers)defput_bucket_logging(self,bucket,logging_xml_doc,headers={}):returnself.generate_url('PUT',bucket,'',{'logging':None},headers)defget_bucket_acl(self,bucket,headers={}):returnself.get_acl(bucket,'',headers)defget_acl(self,bucket,key='',headers={}):returnself.generate_url('GET',bucket,key,{'acl':None},headers)defput_bucket_acl(self,bucket,acl_xml_document,headers={}):returnself.put_acl(bucket,'',acl_xml_document,headers)# don't really care what the doc is here.defput_acl(self,bucket,key,acl_xml_document,headers={}):returnself.generate_url('PUT',bucket,key,{'acl':None},headers)deflist_all_my_buckets(self,headers={}):returnself.generate_url('GET','','',{},headers)defmake_bare_url(self,bucket,key=''):full_url=self.generate_url(self,bucket,key)returnfull_url[:full_url.index('?')]defgenerate_url(self,method,bucket='',key='',query_args={},headers={}):expires=0ifself.__expires_in!=None:expires=int(time.time()+self.__expires_in)elifself.__expires!=None:expires=int(self.__expires)else:raise"Invalid expires state"canonical_str=canonical_string(method,bucket,key,query_args,headers,expires)encoded_canonical=encode(self.aws_secret_access_key,canonical_str)url=CallingFormat.build_url_base(self.protocol,self.server,self.port,bucket,self.calling_format)url+="/%s"%urllib.quote_plus(key)query_args['Signature']=encoded_canonicalquery_args['Expires']=expiresquery_args['AWSAccessKeyId']=self.aws_access_key_idurl+="?%s"%query_args_hash_to_string(query_args)returnurlclassS3Object:def__init__(self,data,metadata={}):self.data=dataself.metadata=metadataclassOwner:def__init__(self,id='',display_name=''):self.id=idself.display_name=display_nameclassListEntry:def__init__(self,key='',last_modified=None,etag='',size=0,storage_class='',owner=None):self.key=keyself.last_modified=last_modifiedself.etag=etagself.size=sizeself.storage_class=storage_classself.owner=ownerclassCommonPrefixEntry:def__init(self,prefix=''):self.prefix=prefixclassBucket:def__init__(self,name='',creation_date=''):self.name=nameself.creation_date=creation_dateclassResponse:def__init__(self,http_response):self.http_response=http_response# you have to do this read, even if you don't expect a body.# otherwise, the next request fails.self.body=http_response.read()ifhttp_response.status>=300andself.body:self.message=self.bodyelse:self.message="%03d%s"%(http_response.status,http_response.reason)classListBucketResponse(Response):def__init__(self,http_response):Response.__init__(self,http_response)ifhttp_response.status<300:handler=ListBucketHandler()xml.sax.parseString(self.body,handler)self.entries=handler.entriesself.common_prefixes=handler.common_prefixesself.name=handler.nameself.marker=handler.markerself.prefix=handler.prefixself.is_truncated=handler.is_truncatedself.delimiter=handler.delimiterself.max_keys=handler.max_keysself.next_marker=handler.next_markerelse:self.entries=[]classListAllMyBucketsResponse(Response):def__init__(self,http_response):Response.__init__(self,http_response)ifhttp_response.status<300:handler=ListAllMyBucketsHandler()xml.sax.parseString(self.body,handler)self.entries=handler.entrieselse:self.entries=[]classGetResponse(Response):def__init__(self,http_response):Response.__init__(self,http_response)response_headers=http_response.msg# older pythons don't have getheadersmetadata=self.get_aws_metadata(response_headers)self.object=S3Object(self.body,metadata)defget_aws_metadata(self,headers):metadata={}forhkeyinheaders.keys():ifhkey.lower().startswith(METADATA_PREFIX):metadata[hkey[len(METADATA_PREFIX):]]=headers[hkey]delheaders[hkey]returnmetadataclassLocationResponse(Response):def__init__(self,http_response):Response.__init__(self,http_response)ifhttp_response.status<300:handler=LocationHandler()xml.sax.parseString(self.body,handler)self.location=handler.locationclassListBucketHandler(xml.sax.ContentHandler):def__init__(self):self.entries=[]self.curr_entry=Noneself.curr_text=''self.common_prefixes=[]self.curr_common_prefix=Noneself.name=''self.marker=''self.prefix=''self.is_truncated=Falseself.delimiter=''self.max_keys=0self.next_marker=''self.is_echoed_prefix_set=FalsedefstartElement(self,name,attrs):ifname=='Contents':self.curr_entry=ListEntry()elifname=='Owner':self.curr_entry.owner=Owner()elifname=='CommonPrefixes':self.curr_common_prefix=CommonPrefixEntry()defendElement(self,name):ifname=='Contents':self.entries.append(self.curr_entry)elifname=='CommonPrefixes':self.common_prefixes.append(self.curr_common_prefix)elifname=='Key':self.curr_entry.key=self.curr_textelifname=='LastModified':self.curr_entry.last_modified=self.curr_textelifname=='ETag':self.curr_entry.etag=self.curr_textelifname=='Size':self.curr_entry.size=int(self.curr_text)elifname=='ID':self.curr_entry.owner.id=self.curr_textelifname=='DisplayName':self.curr_entry.owner.display_name=self.curr_textelifname=='StorageClass':self.curr_entry.storage_class=self.curr_textelifname=='Name':self.name=self.curr_textelifname=='Prefix'andself.is_echoed_prefix_set:self.curr_common_prefix.prefix=self.curr_textelifname=='Prefix':self.prefix=self.curr_textself.is_echoed_prefix_set=Trueelifname=='Marker':self.marker=self.curr_textelifname=='IsTruncated':self.is_truncated=self.curr_text=='true'elifname=='Delimiter':self.delimiter=self.curr_textelifname=='MaxKeys':self.max_keys=int(self.curr_text)elifname=='NextMarker':self.next_marker=self.curr_textself.curr_text=''defcharacters(self,content):self.curr_text+=contentclassListAllMyBucketsHandler(xml.sax.ContentHandler):def__init__(self):self.entries=[]self.curr_entry=Noneself.curr_text=''defstartElement(self,name,attrs):ifname=='Bucket':self.curr_entry=Bucket()defendElement(self,name):ifname=='Name':self.curr_entry.name=self.curr_textelifname=='CreationDate':self.curr_entry.creation_date=self.curr_textelifname=='Bucket':self.entries.append(self.curr_entry)defcharacters(self,content):self.curr_text=contentclassLocationHandler(xml.sax.ContentHandler):def__init__(self):self.location=Noneself.state='init'defstartElement(self,name,attrs):ifself.state=='init':ifname=='LocationConstraint':self.state='tag_location'self.location=''else:self.state='bad'else:self.state='bad'defendElement(self,name):ifself.state=='tag_location'andname=='LocationConstraint':self.state='done'else:self.state='bad'defcharacters(self,content):ifself.state=='tag_location':self.location+=content