# ex:ts=4:sw=4:sts=4:et# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-"""BitBake Utility Functions"""# Copyright (C) 2004 Michael Lauer## This program is free software; you can redistribute it and/or modify# it under the terms of the GNU General Public License version 2 as# published by the Free Software Foundation.## This program is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the# GNU General Public License for more details.## You should have received a copy of the GNU General Public License along# with this program; if not, write to the Free Software Foundation, Inc.,# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.importre,fcntl,os,string,stat,shutil,timeimportsysimporterrnoimportloggingimportbbimportbb.msgimportmultiprocessingimportfcntlimportimpimportitertoolsimportsubprocessimportglobimportfnmatchimporttracebackimporterrnoimportsignalimportastimportcollectionsimportcopyfromsubprocessimportgetstatusoutputfromcontextlibimportcontextmanagerfromctypesimportcdlllogger=logging.getLogger("BitBake.Util")python_extensions=[efore,_,_inimp.get_suffixes()]defclean_context():return{"os":os,"bb":bb,"time":time,}defget_context():return_contextdefset_context(ctx):_context=ctx# Context used in better_exec, eval_context=clean_context()classVersionStringException(Exception):"""Exception raised when an invalid version specification is found"""defexplode_version(s):r=[]alpha_regexp=re.compile('^([a-zA-Z]+)(.*)$')numeric_regexp=re.compile('^(\d+)(.*)$')while(s!=''):ifs[0]instring.digits:m=numeric_regexp.match(s)r.append((0,int(m.group(1))))s=m.group(2)continueifs[0]instring.ascii_letters:m=alpha_regexp.match(s)r.append((1,m.group(1)))s=m.group(2)continueifs[0]=='~':r.append((-1,s[0]))else:r.append((2,s[0]))s=s[1:]returnrdefsplit_version(s):"""Split a version string into its constituent parts (PE, PV, PR)"""s=s.strip(" <>=")e=0ifs.count(':'):e=int(s.split(":")[0])s=s.split(":")[1]r=""ifs.count('-'):r=s.rsplit("-",1)[1]s=s.rsplit("-",1)[0]v=sreturn(e,v,r)defvercmp_part(a,b):va=explode_version(a)vb=explode_version(b)whileTrue:ifva==[]:(oa,ca)=(0,None)else:(oa,ca)=va.pop(0)ifvb==[]:(ob,cb)=(0,None)else:(ob,cb)=vb.pop(0)if(oa,ca)==(0,None)and(ob,cb)==(0,None):return0ifoa<ob:return-1elifoa>ob:return1elifca<cb:return-1elifca>cb:return1defvercmp(ta,tb):(ea,va,ra)=ta(eb,vb,rb)=tbr=int(eaor0)-int(ebor0)if(r==0):r=vercmp_part(va,vb)if(r==0):r=vercmp_part(ra,rb)returnrdefvercmp_string(a,b):ta=split_version(a)tb=split_version(b)returnvercmp(ta,tb)defvercmp_string_op(a,b,op):""" Compare two versions and check if the specified comparison operator matches the result of the comparison. This function is fairly liberal about what operators it will accept since there are a variety of styles depending on the context. """res=vercmp_string(a,b)ifopin('=','=='):returnres==0elifop=='<=':returnres<=0elifop=='>=':returnres>=0elifopin('>','>>'):returnres>0elifopin('<','<<'):returnres<0elifop=='!=':returnres!=0else:raiseVersionStringException('Unsupported comparison operator "%s"'%op)defexplode_deps(s):""" Take an RDEPENDS style string of format: "DEPEND1 (optional version) DEPEND2 (optional version) ..." and return a list of dependencies. Version information is ignored. """r=[]l=s.split()flag=Falseforiinl:ifi[0]=='(':flag=True#j = []ifnotflag:r.append(i)#else:# j.append(i)ifflagandi.endswith(')'):flag=False# Ignore version#r[-1] += ' ' + ' '.join(j)returnrdefexplode_dep_versions2(s):""" Take an RDEPENDS style string of format: "DEPEND1 (optional version) DEPEND2 (optional version) ..." and return a dictionary of dependencies and versions. """r=collections.OrderedDict()l=s.replace(",","").split()lastdep=Nonelastcmp=""lastver=""incmp=Falseinversion=Falseforiinl:ifi[0]=='(':incmp=Truei=i[1:].strip()ifnoti:continueifincmp:incmp=Falseinversion=True# This list is based on behavior and supported comparisons from deb, opkg and rpm.## Even though =<, <<, ==, !=, =>, and >> may not be supported, # we list each possibly valid item. # The build system is responsible for validation of what it supports.ifi.startswith(('<=','=<','<<','==','!=','>=','=>','>>')):lastcmp=i[0:2]i=i[2:]elifi.startswith(('<','>','=')):lastcmp=i[0:1]i=i[1:]else:# This is an unsupported case!raiseVersionStringException('Invalid version specification in "(%s" - invalid or missing operator'%i)lastcmp=(ior"")i=""i.strip()ifnoti:continueifinversion:ifi.endswith(')'):i=i[:-1]or""inversion=Falseiflastverandi:lastver+=" "ifi:lastver+=iiflastdepnotinr:r[lastdep]=[]r[lastdep].append(lastcmp+" "+lastver)continue#if not inversion:lastdep=ilastver=""lastcmp=""ifnot(iinrandr[i]):r[lastdep]=[]r=collections.OrderedDict(sorted(r.items(),key=lambdax:x[0]))returnrdefexplode_dep_versions(s):r=explode_dep_versions2(s)fordinr:ifnotr[d]:r[d]=Nonecontinueiflen(r[d])>1:bb.warn("explode_dep_versions(): Item %s appeared in dependency string '%s' multiple times with different values. explode_dep_versions cannot cope with this."%(d,s))r[d]=r[d][0]returnrdefjoin_deps(deps,commasep=True):""" Take the result from explode_dep_versions and generate a dependency string """result=[]fordepindeps:ifdeps[dep]:ifisinstance(deps[dep],list):forvindeps[dep]:result.append(dep+" ("+v+")")else:result.append(dep+" ("+deps[dep]+")")else:result.append(dep)ifcommasep:return", ".join(result)else:return" ".join(result)def_print_trace(body,line):""" Print the Environment of a Text Body """error=[]# print the environment of the methodmin_line=max(1,line-4)max_line=min(line+4,len(body))foriinrange(min_line,max_line+1):ifline==i:error.append(' *** %.4d:%s'%(i,body[i-1].rstrip()))else:error.append(' %.4d:%s'%(i,body[i-1].rstrip()))returnerrordefbetter_compile(text,file,realfile,mode="exec",lineno=0):""" A better compile method. This method will print the offending lines. """try:cache=bb.methodpool.compile_cache(text)ifcache:returncache# We can't add to the linenumbers for compile, we can pad to the correct number of blank lines thoughtext2="\n"*int(lineno)+textcode=compile(text2,realfile,mode)bb.methodpool.compile_cache_add(text,code)returncodeexceptExceptionase:error=[]# split the text into lines againbody=text.split('\n')error.append("Error in compiling python function in %s, line %s:\n"%(realfile,lineno))ifhasattr(e,"lineno"):error.append("The code lines resulting in this error were:")error.extend(_print_trace(body,e.lineno))else:error.append("The function causing this error was:")forlineinbody:error.append(line)error.append("%s: %s"%(e.__class__.__name__,str(e)))logger.error("\n".join(error))e=bb.BBHandledException(e)raiseedef_print_exception(t,value,tb,realfile,text,context):error=[]try:exception=traceback.format_exception_only(t,value)error.append('Error executing a python function in %s:\n'%realfile)# Strip 'us' from the stack (better_exec call) unless that was where the # error came fromiftb.tb_nextisnotNone:tb=tb.tb_nexttextarray=text.split('\n')linefailed=tb.tb_linenotbextract=traceback.extract_tb(tb)tbformat=traceback.format_list(tbextract)error.append("The stack trace of python calls that resulted in this exception/failure was:")error.append("File: '%s', lineno: %s, function: %s"%(tbextract[0][0],tbextract[0][1],tbextract[0][2]))error.extend(_print_trace(textarray,linefailed))# See if this is a function we constructed and has calls back into other functions in# "text". If so, try and improve the context of the error by diving down the tracelevel=0nexttb=tb.tb_nextwhilenexttbisnotNoneand(level+1)<len(tbextract):error.append("File: '%s', lineno: %s, function: %s"%(tbextract[level+1][0],tbextract[level+1][1],tbextract[level+1][2]))iftbextract[level][0]==tbextract[level+1][0]andtbextract[level+1][2]==tbextract[level][0]:# The code was possibly in the string we compiled ourselveserror.extend(_print_trace(textarray,tbextract[level+1][1]))eliftbextract[level+1][0].startswith("/"):# The code looks like it might be in a file, try and load ittry:withopen(tbextract[level+1][0],"r")asf:text=f.readlines()error.extend(_print_trace(text,tbextract[level+1][1]))except:error.append(tbformat[level+1])else:error.append(tbformat[level+1])nexttb=tb.tb_nextlevel=level+1error.append("Exception: %s"%''.join(exception))# If the exception is from spwaning a task, let's be helpful and display# the output (which hopefully includes stderr).ifisinstance(value,subprocess.CalledProcessError)andvalue.output:error.append("Subprocess output:")error.append(value.output.decode("utf-8",errors="ignore"))finally:logger.error("\n".join(error))defbetter_exec(code,context,text=None,realfile="<code>",pythonexception=False):""" Similiar to better_compile, better_exec will print the lines that are responsible for the error. """importbb.parseifnottext:text=codeifnothasattr(code,"co_filename"):code=better_compile(code,realfile,realfile)try:exec(code,get_context(),context)except(bb.BBHandledException,bb.parse.SkipRecipe,bb.build.FuncFailed,bb.data_smart.ExpansionError):# Error already shown so passthrough, no need for tracebackraiseexceptExceptionase:ifpythonexception:raise(t,value,tb)=sys.exc_info()try:_print_exception(t,value,tb,realfile,text,context)exceptExceptionase:logger.error("Exception handler error: %s"%str(e))e=bb.BBHandledException(e)raiseedefsimple_exec(code,context):exec(code,get_context(),context)defbetter_eval(source,locals,extraglobals=None):ctx=get_context()ifextraglobals:ctx=copy.copy(ctx)forginextraglobals:ctx[g]=extraglobals[g]returneval(source,ctx,locals)@contextmanagerdeffileslocked(files):"""Context manager for locking and unlocking file locks."""locks=[]iffiles:forlockfileinfiles:locks.append(bb.utils.lockfile(lockfile))yieldforlockinlocks:bb.utils.unlockfile(lock)@contextmanagerdeftimeout(seconds):deftimeout_handler(signum,frame):passoriginal_handler=signal.signal(signal.SIGALRM,timeout_handler)try:signal.alarm(seconds)yieldfinally:signal.alarm(0)signal.signal(signal.SIGALRM,original_handler)deflockfile(name,shared=False,retry=True,block=False):""" Use the specified file as a lock file, return when the lock has been acquired. Returns a variable to pass to unlockfile(). Parameters: retry: True to re-try locking if it fails, False otherwise block: True to block until the lock succeeds, False otherwise The retry and block parameters are kind of equivalent unless you consider the possibility of sending a signal to the process to break out - at which point you want block=True rather than retry=True. """dirname=os.path.dirname(name)mkdirhier(dirname)ifnotos.access(dirname,os.W_OK):logger.error("Unable to acquire lock '%s', directory is not writable",name)sys.exit(1)op=fcntl.LOCK_EXifshared:op=fcntl.LOCK_SHifnotretryandnotblock:op=op|fcntl.LOCK_NBwhileTrue:# If we leave the lockfiles lying around there is no problem# but we should clean up after ourselves. This gives potential# for races though. To work around this, when we acquire the lock# we check the file we locked was still the lock file on disk.# by comparing inode numbers. If they don't match or the lockfile# no longer exists, we start again.# This implementation is unfair since the last person to request the# lock is the most likely to win it.try:lf=open(name,'a+')fileno=lf.fileno()fcntl.flock(fileno,op)statinfo=os.fstat(fileno)ifos.path.exists(lf.name):statinfo2=os.stat(lf.name)ifstatinfo.st_ino==statinfo2.st_ino:returnlflf.close()exceptException:try:lf.close()exceptException:passpassifnotretry:returnNonedefunlockfile(lf):""" Unlock a file locked using lockfile() """try:# If we had a shared lock, we need to promote to exclusive before# removing the lockfile. Attempt this, ignore failures.fcntl.flock(lf.fileno(),fcntl.LOCK_EX|fcntl.LOCK_NB)os.unlink(lf.name)except(IOError,OSError):passfcntl.flock(lf.fileno(),fcntl.LOCK_UN)lf.close()defmd5_file(filename):""" Return the hex string representation of the MD5 checksum of filename. """importhashlibm=hashlib.md5()withopen(filename,"rb")asf:forlineinf:m.update(line)returnm.hexdigest()defsha256_file(filename):""" Return the hex string representation of the 256-bit SHA checksum of filename. """importhashlibs=hashlib.sha256()withopen(filename,"rb")asf:forlineinf:s.update(line)returns.hexdigest()defsha1_file(filename):""" Return the hex string representation of the SHA1 checksum of the filename """importhashlibs=hashlib.sha1()withopen(filename,"rb")asf:forlineinf:s.update(line)returns.hexdigest()defpreserved_envvars_exported():"""Variables which are taken from the environment and placed in and exported from the metadata"""return['BB_TASKHASH','HOME','LOGNAME','PATH','PWD','SHELL','TERM','USER','LC_ALL','BBSERVER',]defpreserved_envvars():"""Variables which are taken from the environment and placed in the metadata"""v=['BBPATH','BB_PRESERVE_ENV','BB_ENV_WHITELIST','BB_ENV_EXTRAWHITE',]returnv+preserved_envvars_exported()deffilter_environment(good_vars):""" Create a pristine environment for bitbake. This will remove variables that are not known and may influence the build in a negative way. """removed_vars={}forkeyinlist(os.environ):ifkeyingood_vars:continueremoved_vars[key]=os.environ[key]delos.environ[key]# If we spawn a python process, we need to have a UTF-8 locale, else python's file# access methods will use ascii. You can't change that mode once the interpreter is# started so we have to ensure a locale is set. Ideally we'd use C.UTF-8 but not all# distros support that and we need to set something.os.environ["LC_ALL"]="en_US.UTF-8"ifremoved_vars:logger.debug(1,"Removed the following variables from the environment: %s",", ".join(removed_vars.keys()))returnremoved_varsdefapproved_variables():""" Determine and return the list of whitelisted variables which are approved to remain in the environment. """if'BB_PRESERVE_ENV'inos.environ:returnos.environ.keys()approved=[]if'BB_ENV_WHITELIST'inos.environ:approved=os.environ['BB_ENV_WHITELIST'].split()approved.extend(['BB_ENV_WHITELIST'])else:approved=preserved_envvars()if'BB_ENV_EXTRAWHITE'inos.environ:approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())if'BB_ENV_EXTRAWHITE'notinapproved:approved.extend(['BB_ENV_EXTRAWHITE'])returnapproveddefclean_environment():""" Clean up any spurious environment variables. This will remove any variables the user hasn't chosen to preserve. """if'BB_PRESERVE_ENV'notinos.environ:good_vars=approved_variables()returnfilter_environment(good_vars)return{}defempty_environment():""" Remove all variables from the environment. """forsinlist(os.environ.keys()):os.unsetenv(s)delos.environ[s]defbuild_environment(d):""" Build an environment from all exported variables. """importbb.dataforvarinbb.data.keys(d):export=d.getVarFlag(var,"export",False)ifexport:os.environ[var]=d.getVar(var)or""def_check_unsafe_delete_path(path):""" Basic safeguard against recursively deleting something we shouldn't. If it returns True, the caller should raise an exception with an appropriate message. NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes with potentially disastrous results. """extra=''# HOME might not be /home/something, so in case we can get it, check against ithomedir=os.environ.get('HOME','')ifhomedir:extra='|%s'%homedirifre.match('(/|//|/home|/home/[^/]*%s)$'%extra,os.path.abspath(path)):returnTruereturnFalsedefremove(path,recurse=False):"""Equivalent to rm -f or rm -rf"""ifnotpath:returnifrecurse:fornameinglob.glob(path):if_check_unsafe_delete_path(path):raiseException('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!'%path)# shutil.rmtree(name) would be ideal but its too slowsubprocess.check_call(['rm','-rf']+glob.glob(path))returnfornameinglob.glob(path):try:os.unlink(name)exceptOSErrorasexc:ifexc.errno!=errno.ENOENT:raisedefprunedir(topdir):# Delete everything reachable from the directory named in 'topdir'.# CAUTION: This is dangerous!if_check_unsafe_delete_path(topdir):raiseException('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!'%topdir)forroot,dirs,filesinos.walk(topdir,topdown=False):fornameinfiles:os.remove(os.path.join(root,name))fornameindirs:ifos.path.islink(os.path.join(root,name)):os.remove(os.path.join(root,name))else:os.rmdir(os.path.join(root,name))os.rmdir(topdir)## Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)# but thats possibly insane and suffixes is probably going to be small#defprune_suffix(var,suffixes,d):# See if var ends with any of the suffixes listed and# remove it if foundforsuffixinsuffixes:ifvar.endswith(suffix):returnvar.replace(suffix,"")returnvardefmkdirhier(directory):"""Create a directory like 'mkdir -p', but does not complain if directory already exists like os.makedirs """try:os.makedirs(directory)exceptOSErrorase:ife.errno!=errno.EEXIST:raiseedefmovefile(src,dest,newmtime=None,sstat=None):"""Moves a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. Move is atomic. """#print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"try:ifnotsstat:sstat=os.lstat(src)exceptExceptionase:print("movefile: Stating source file failed...",e)returnNonedestexists=1try:dstat=os.lstat(dest)except:dstat=os.lstat(os.path.dirname(dest))destexists=0ifdestexists:ifstat.S_ISLNK(dstat[stat.ST_MODE]):try:os.unlink(dest)destexists=0exceptExceptionase:passifstat.S_ISLNK(sstat[stat.ST_MODE]):try:target=os.readlink(src)ifdestexistsandnotstat.S_ISDIR(dstat[stat.ST_MODE]):os.unlink(dest)os.symlink(target,dest)#os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])os.unlink(src)returnos.lstat(dest)exceptExceptionase:print("movefile: failed to properly create symlink:",dest,"->",target,e)returnNonerenamefailed=1ifsstat[stat.ST_DEV]==dstat[stat.ST_DEV]:try:# os.rename needs to know the dest path ending with file name# so append the file name to a path only if it's a dir specifiedsrcfname=os.path.basename(src)destpath=os.path.join(dest,srcfname)ifos.path.isdir(dest) \
elsedestos.rename(src,destpath)renamefailed=0exceptExceptionase:ife[0]!=errno.EXDEV:# Some random error.print("movefile: Failed to move",src,"to",dest,e)returnNone# Invalid cross-device-link 'bind' mounted or actually Cross-Deviceifrenamefailed:didcopy=0ifstat.S_ISREG(sstat[stat.ST_MODE]):try:# For safety copy then move it over.shutil.copyfile(src,dest+"#new")os.rename(dest+"#new",dest)didcopy=1exceptExceptionase:print('movefile: copy',src,'->',dest,'failed.',e)returnNoneelse:#we don't yet handle special, so we need to fall back to /bin/mva=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'")ifa[0]!=0:print("movefile: Failed to move special file:"+src+"' to '"+dest+"'",a)returnNone# failuretry:ifdidcopy:os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])os.chmod(dest,stat.S_IMODE(sstat[stat.ST_MODE]))# Sticky is reset on chownos.unlink(src)exceptExceptionase:print("movefile: Failed to chown/chmod/unlink",dest,e)returnNoneifnewmtime:os.utime(dest,(newmtime,newmtime))else:os.utime(dest,(sstat[stat.ST_ATIME],sstat[stat.ST_MTIME]))newmtime=sstat[stat.ST_MTIME]returnnewmtimedefcopyfile(src,dest,newmtime=None,sstat=None):""" Copies a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. """#print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"try:ifnotsstat:sstat=os.lstat(src)exceptExceptionase:logger.warning("copyfile: stat of %s failed (%s)"%(src,e))returnFalsedestexists=1try:dstat=os.lstat(dest)except:dstat=os.lstat(os.path.dirname(dest))destexists=0ifdestexists:ifstat.S_ISLNK(dstat[stat.ST_MODE]):try:os.unlink(dest)destexists=0exceptExceptionase:passifstat.S_ISLNK(sstat[stat.ST_MODE]):try:target=os.readlink(src)ifdestexistsandnotstat.S_ISDIR(dstat[stat.ST_MODE]):os.unlink(dest)os.symlink(target,dest)#os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])returnos.lstat(dest)exceptExceptionase:logger.warning("copyfile: failed to create symlink %s to %s (%s)"%(dest,target,e))returnFalseifstat.S_ISREG(sstat[stat.ST_MODE]):try:srcchown=Falseifnotos.access(src,os.R_OK):# Make sure we can read itsrcchown=Trueos.chmod(src,sstat[stat.ST_MODE]|stat.S_IRUSR)# For safety copy then move it over.shutil.copyfile(src,dest+"#new")os.rename(dest+"#new",dest)exceptExceptionase:logger.warning("copyfile: copy %s to %s failed (%s)"%(src,dest,e))returnFalsefinally:ifsrcchown:os.chmod(src,sstat[stat.ST_MODE])os.utime(src,(sstat[stat.ST_ATIME],sstat[stat.ST_MTIME]))else:#we don't yet handle special, so we need to fall back to /bin/mva=getstatusoutput("/bin/cp -f "+"'"+src+"' '"+dest+"'")ifa[0]!=0:logger.warning("copyfile: failed to copy special file %s to %s (%s)"%(src,dest,a))returnFalse# failuretry:os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])os.chmod(dest,stat.S_IMODE(sstat[stat.ST_MODE]))# Sticky is reset on chownexceptExceptionase:logger.warning("copyfile: failed to chown/chmod %s (%s)"%(dest,e))returnFalseifnewmtime:os.utime(dest,(newmtime,newmtime))else:os.utime(dest,(sstat[stat.ST_ATIME],sstat[stat.ST_MTIME]))newmtime=sstat[stat.ST_MTIME]returnnewmtimedefwhich(path,item,direction=0,history=False,executable=False):""" Locate `item` in the list of paths `path` (colon separated string like $PATH). If `direction` is non-zero then the list is reversed. If `history` is True then the list of candidates also returned as result,history. If `executable` is True then the candidate has to be an executable file, otherwise the candidate simply has to exist. """ifexecutable:is_candidate=lambdap:os.path.isfile(p)andos.access(p,os.X_OK)else:is_candidate=lambdap:os.path.exists(p)hist=[]paths=(pathor"").split(':')ifdirection!=0:paths.reverse()forpinpaths:next=os.path.join(p,item)hist.append(next)ifis_candidate(next):ifnotos.path.isabs(next):next=os.path.abspath(next)ifhistory:returnnext,histreturnnextifhistory:return"",histreturn""defto_boolean(string,default=None):ifnotstring:returndefaultnormalized=string.lower()ifnormalizedin("y","yes","1","true"):returnTrueelifnormalizedin("n","no","0","false"):returnFalseelse:raiseValueError("Invalid value for to_boolean: %s"%string)defcontains(variable,checkvalues,truevalue,falsevalue,d):"""Check if a variable contains all the values specified. Arguments: variable -- the variable name. This will be fetched and expanded (using d.getVar(variable)) and then split into a set(). checkvalues -- if this is a string it is split on whitespace into a set(), otherwise coerced directly into a set(). truevalue -- the value to return if checkvalues is a subset of variable. falsevalue -- the value to return if variable is empty or if checkvalues is not a subset of variable. d -- the data store. """val=d.getVar(variable)ifnotval:returnfalsevalueval=set(val.split())ifisinstance(checkvalues,str):checkvalues=set(checkvalues.split())else:checkvalues=set(checkvalues)ifcheckvalues.issubset(val):returntruevaluereturnfalsevaluedefcontains_any(variable,checkvalues,truevalue,falsevalue,d):val=d.getVar(variable)ifnotval:returnfalsevalueval=set(val.split())ifisinstance(checkvalues,str):checkvalues=set(checkvalues.split())else:checkvalues=set(checkvalues)ifcheckvalues&val:returntruevaluereturnfalsevaluedeffilter(variable,checkvalues,d):"""Return all words in the variable that are present in the checkvalues. Arguments: variable -- the variable name. This will be fetched and expanded (using d.getVar(variable)) and then split into a set(). checkvalues -- if this is a string it is split on whitespace into a set(), otherwise coerced directly into a set(). d -- the data store. """val=d.getVar(variable)ifnotval:return''val=set(val.split())ifisinstance(checkvalues,str):checkvalues=set(checkvalues.split())else:checkvalues=set(checkvalues)return' '.join(sorted(checkvalues&val))defcpu_count():returnmultiprocessing.cpu_count()defnonblockingfd(fd):fcntl.fcntl(fd,fcntl.F_SETFL,fcntl.fcntl(fd,fcntl.F_GETFL)|os.O_NONBLOCK)defprocess_profilelog(fn,pout=None):# Either call with a list of filenames and set pout or a filename and optionally pout.ifnotpout:pout=fn+'.processed'pout=open(pout,'w')importpstatsifisinstance(fn,list):p=pstats.Stats(*fn,stream=pout)else:p=pstats.Stats(fn,stream=pout)p.sort_stats('time')p.print_stats()p.print_callers()p.sort_stats('cumulative')p.print_stats()pout.flush()pout.close()## Was present to work around multiprocessing pool bugs in python < 2.7.3#defmultiprocessingpool(*args,**kwargs):importmultiprocessing.pool#import multiprocessing.util#multiprocessing.util.log_to_stderr(10)# Deal with a multiprocessing bug where signals to the processes would be delayed until the work# completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.defwrapper(func):defwrap(self,timeout=None):returnfunc(self,timeout=timeoutiftimeoutisnotNoneelse1e100)returnwrapmultiprocessing.pool.IMapIterator.next=wrapper(multiprocessing.pool.IMapIterator.next)returnmultiprocessing.Pool(*args,**kwargs)defexec_flat_python_func(func,*args,**kwargs):"""Execute a flat python function (defined with def funcname(args):...)"""# Prepare a small piece of python code which calls the requested function# To do this we need to prepare two things - a set of variables we can use to pass# the values of arguments into the calling function, and the list of arguments for# the function being calledcontext={}funcargs=[]# Handle unnamed argumentsaidx=1forarginargs:argname='arg_%s'%aidxcontext[argname]=argfuncargs.append(argname)aidx+=1# Handle keyword argumentscontext.update(kwargs)funcargs.extend(['%s=%s'%(arg,arg)forarginkwargs.keys()])code='retval = %s(%s)'%(func,', '.join(funcargs))comp=bb.utils.better_compile(code,'<string>','<string>')bb.utils.better_exec(comp,context,code,'<string>')returncontext['retval']defedit_metadata(meta_lines,variables,varfunc,match_overrides=False):"""Edit lines from a recipe or config file and modify one or more specified variable values set in the file using a specified callback function. Lines are expected to have trailing newlines. Parameters: meta_lines: lines from the file; can be a list or an iterable (e.g. file pointer) variables: a list of variable names to look for. Functions may also be specified, but must be specified with '()' at the end of the name. Note that the function doesn't have any intrinsic understanding of _append, _prepend, _remove, or overrides, so these are considered as part of the name. These values go into a regular expression, so regular expression syntax is allowed. varfunc: callback function called for every variable matching one of the entries in the variables parameter. The function should take four arguments: varname: name of variable matched origvalue: current value in file op: the operator (e.g. '+=') newlines: list of lines up to this point. You can use this to prepend lines before this variable setting if you wish. and should return a four-element tuple: newvalue: new value to substitute in, or None to drop the variable setting entirely. (If the removal results in two consecutive blank lines, one of the blank lines will also be dropped). newop: the operator to use - if you specify None here, the original operation will be used. indent: number of spaces to indent multi-line entries, or -1 to indent up to the level of the assignment and opening quote, or a string to use as the indent. minbreak: True to allow the first element of a multi-line value to continue on the same line as the assignment, False to indent before the first element. To clarify, if you wish not to change the value, then you would return like this: return origvalue, None, 0, True match_overrides: True to match items with _overrides on the end, False otherwise Returns a tuple: updated: True if changes were made, False otherwise. newlines: Lines after processing """var_res={}ifmatch_overrides:override_re='(_[a-zA-Z0-9-_$(){}]+)?'else:override_re=''forvarinvariables:ifvar.endswith('()'):var_res[var]=re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{'%(var[:-2].rstrip(),override_re))else:var_res[var]=re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])'%(var,override_re))updated=Falsevarset_start=''varlines=[]newlines=[]in_var=Nonefull_value=''var_end=''defhandle_var_end():prerun_newlines=newlines[:]op=varset_start[len(in_var):].strip()(newvalue,newop,indent,minbreak)=varfunc(in_var,full_value,op,newlines)changed=(prerun_newlines!=newlines)ifnewvalueisNone:# Drop the valuereturnTrueelifnewvalue!=full_valueor(newopnotin[None,op]):ifnewopnotin[None,op]:# Callback changed the operatorvarset_new="%s%s"%(in_var,newop)else:varset_new=varset_startifisinstance(indent,int):ifindent==-1:indentspc=' '*(len(varset_new)+2)else:indentspc=' '*indentelse:indentspc=indentifin_var.endswith('()'):# A function definitionifisinstance(newvalue,list):newlines.append('%s {\n%s%s\n}\n'%(varset_new,indentspc,('\n%s'%indentspc).join(newvalue)))else:ifnotnewvalue.startswith('\n'):newvalue='\n'+newvalueifnotnewvalue.endswith('\n'):newvalue=newvalue+'\n'newlines.append('%s {%s}\n'%(varset_new,newvalue))else:# Normal variableifisinstance(newvalue,list):ifnotnewvalue:# Empty list -> empty stringnewlines.append('%s ""\n'%varset_new)elifminbreak:# First item on first lineiflen(newvalue)==1:newlines.append('%s "%s"\n'%(varset_new,newvalue[0]))else:newlines.append('%s "%s\\\n'%(varset_new,newvalue[0]))foriteminnewvalue[1:]:newlines.append('%s%s\\\n'%(indentspc,item))newlines.append('%s"\n'%indentspc)else:# No item on first linenewlines.append('%s " \\\n'%varset_new)foriteminnewvalue:newlines.append('%s%s\\\n'%(indentspc,item))newlines.append('%s"\n'%indentspc)else:newlines.append('%s "%s"\n'%(varset_new,newvalue))returnTrueelse:# Put the old lines back where they werenewlines.extend(varlines)# If newlines was touched by the function, we'll need to return Truereturnchangedcheckspc=Falseforlineinmeta_lines:ifin_var:value=line.rstrip()varlines.append(line)ifin_var.endswith('()'):full_value+='\n'+valueelse:full_value+=value[:-1]ifvalue.endswith(var_end):ifin_var.endswith('()'):iffull_value.count('{')-full_value.count('}')>=0:continuefull_value=full_value[:-1]ifhandle_var_end():updated=Truecheckspc=Truein_var=Noneelse:skip=Falsefor(varname,var_re)invar_res.items():res=var_re.match(line)ifres:isfunc=varname.endswith('()')ifisfunc:splitvalue=line.split('{',1)var_end='}'else:var_end=res.groups()[-1]splitvalue=line.split(var_end,1)varset_start=splitvalue[0].rstrip()value=splitvalue[1].rstrip()ifnotisfuncandvalue.endswith('\\'):value=value[:-1]full_value=valuevarlines=[line]in_var=res.group(1)ifisfunc:in_var+='()'ifvalue.endswith(var_end):full_value=full_value[:-1]ifhandle_var_end():updated=Truecheckspc=Truein_var=Noneskip=Truebreakifnotskip:ifcheckspc:checkspc=Falseifnewlinesandnewlines[-1]=='\n'andline=='\n':# Squash blank line if there are two consecutive blanks after a removalcontinuenewlines.append(line)return(updated,newlines)defedit_metadata_file(meta_file,variables,varfunc):"""Edit a recipe or config file and modify one or more specified variable values set in the file using a specified callback function. The file is only written to if the value(s) actually change. This is basically the file version of edit_metadata(), see that function's description for parameter/usage information. Returns True if the file was written to, False otherwise. """withopen(meta_file,'r')asf:(updated,newlines)=edit_metadata(f,variables,varfunc)ifupdated:withopen(meta_file,'w')asf:f.writelines(newlines)returnupdateddefedit_bblayers_conf(bblayers_conf,add,remove):"""Edit bblayers.conf, adding and/or removing layers Parameters: bblayers_conf: path to bblayers.conf file to edit add: layer path (or list of layer paths) to add; None or empty list to add nothing remove: layer path (or list of layer paths) to remove; None or empty list to remove nothing Returns a tuple: notadded: list of layers specified to be added but weren't (because they were already in the list) notremoved: list of layers that were specified to be removed but weren't (because they weren't in the list) """importfnmatchdefremove_trailing_sep(pth):ifpthandpth[-1]==os.sep:pth=pth[:-1]returnpthapproved=bb.utils.approved_variables()defcanonicalise_path(pth):pth=remove_trailing_sep(pth)if'HOME'inapprovedand'~'inpth:pth=os.path.expanduser(pth)returnpthdeflayerlist_param(value):ifnotvalue:return[]elifisinstance(value,list):return[remove_trailing_sep(x)forxinvalue]else:return[remove_trailing_sep(value)]addlayers=layerlist_param(add)removelayers=layerlist_param(remove)# Need to use a list here because we can't set non-local variables from a callback in python 2.xbblayercalls=[]removed=[]plusequals=Falseorig_bblayers=[]defhandle_bblayers_firstpass(varname,origvalue,op,newlines):bblayercalls.append(op)ifop=='=':delorig_bblayers[:]orig_bblayers.extend([canonicalise_path(x)forxinorigvalue.split()])return(origvalue,None,2,False)defhandle_bblayers(varname,origvalue,op,newlines):updated=Falsebblayers=[remove_trailing_sep(x)forxinorigvalue.split()]ifremovelayers:forremovelayerinremovelayers:forlayerinbblayers:iffnmatch.fnmatch(canonicalise_path(layer),canonicalise_path(removelayer)):updated=Truebblayers.remove(layer)removed.append(removelayer)breakifaddlayersandnotplusequals:foraddlayerinaddlayers:ifaddlayernotinbblayers:updated=Truebblayers.append(addlayer)deladdlayers[:]ifupdated:ifop=='+='andnotbblayers:bblayers=Nonereturn(bblayers,None,2,False)else:return(origvalue,None,2,False)withopen(bblayers_conf,'r')asf:(_,newlines)=edit_metadata(f,['BBLAYERS'],handle_bblayers_firstpass)ifnotbblayercalls:raiseException('Unable to find BBLAYERS in %s'%bblayers_conf)# Try to do the "smart" thing depending on how the user has laid out# their bblayers.conf fileifbblayercalls.count('+=')>1:plusequals=Trueremovelayers_canon=[canonicalise_path(layer)forlayerinremovelayers]notadded=[]forlayerinaddlayers:layer_canon=canonicalise_path(layer)iflayer_canoninorig_bblayersandnotlayer_canoninremovelayers_canon:notadded.append(layer)notadded_canon=[canonicalise_path(layer)forlayerinnotadded]addlayers[:]=[layerforlayerinaddlayersifcanonicalise_path(layer)notinnotadded_canon](updated,newlines)=edit_metadata(newlines,['BBLAYERS'],handle_bblayers)ifaddlayers:# Still need to add theseforaddlayerinaddlayers:newlines.append('BBLAYERS += "%s"\n'%addlayer)updated=Trueifupdated:withopen(bblayers_conf,'w')asf:f.writelines(newlines)notremoved=list(set(removelayers)-set(removed))return(notadded,notremoved)defget_file_layer(filename,d):"""Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""collections=(d.getVar('BBFILE_COLLECTIONS')or'').split()collection_res={}forcollectionincollections:collection_res[collection]=d.getVar('BBFILE_PATTERN_%s'%collection)or''defpath_to_layer(path):# Use longest path so we handle nested layersmatchlen=0match=Noneforcollection,regexincollection_res.items():iflen(regex)>matchlenandre.match(regex,path):matchlen=len(regex)match=collectionreturnmatchresult=Nonebbfiles=(d.getVar('BBFILES')or'').split()bbfilesmatch=Falseforbbfilesentryinbbfiles:iffnmatch.fnmatch(filename,bbfilesentry):bbfilesmatch=Trueresult=path_to_layer(bbfilesentry)ifnotbbfilesmatch:# Probably a bbclassresult=path_to_layer(filename)returnresult# Constant taken from http://linux.die.net/include/linux/prctl.hPR_SET_PDEATHSIG=1classPrCtlError(Exception):passdefsignal_on_parent_exit(signame):""" Trigger signame to be sent when the parent process dies """signum=getattr(signal,signame)# http://linux.die.net/man/2/prctlresult=cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG,signum)ifresult!=0:raisePrCtlError('prctl failed with error code %s'%result)## Manually call the ioprio syscall. We could depend on other libs like psutil# however this gets us enough of what we need to bitbake for now without the# dependency#_unamearch=os.uname()[4]IOPRIO_WHO_PROCESS=1IOPRIO_CLASS_SHIFT=13defioprio_set(who,cls,value):NR_ioprio_set=Noneif_unamearch=="x86_64":NR_ioprio_set=251elif_unamearch[0]=="i"and_unamearch[2:3]=="86":NR_ioprio_set=289ifNR_ioprio_set:ioprio=value|(cls<<IOPRIO_CLASS_SHIFT)rc=cdll['libc.so.6'].syscall(NR_ioprio_set,IOPRIO_WHO_PROCESS,who,ioprio)ifrc!=0:raiseValueError("Unable to set ioprio, syscall returned %s"%rc)else:bb.warn("Unable to set IO Prio for arch %s"%_unamearch)defset_process_name(name):fromctypesimportcdll,byref,create_string_buffer# This is nice to have for debugging, not essentialtry:libc=cdll.LoadLibrary('libc.so.6')buf=create_string_buffer(bytes(name,'utf-8'))libc.prctl(15,byref(buf),0,0,0)except:pass# export common proxies variables from datastore to environmentdefexport_proxies(d):importosvariables=['http_proxy','HTTP_PROXY','https_proxy','HTTPS_PROXY','ftp_proxy','FTP_PROXY','no_proxy','NO_PROXY','GIT_PROXY_COMMAND']exported=Falseforvinvariables:ifvinos.environ.keys():exported=Trueelse:v_proxy=d.getVar(v)ifv_proxyisnotNone:os.environ[v]=v_proxyexported=Truereturnexporteddefload_plugins(logger,plugins,pluginpath):defload_plugin(name):logger.debug(1,'Loading plugin %s'%name)fp,pathname,description=imp.find_module(name,[pluginpath])try:returnimp.load_module(name,fp,pathname,description)finally:iffp:fp.close()logger.debug(1,'Loading plugins from %s...'%pluginpath)expanded=(glob.glob(os.path.join(pluginpath,'*'+ext))forextinpython_extensions)files=itertools.chain.from_iterable(expanded)names=set(os.path.splitext(os.path.basename(fn))[0]forfninfiles)fornameinnames:ifname!='__init__':plugin=load_plugin(name)ifhasattr(plugin,'plugin_init'):obj=plugin.plugin_init(plugins)plugins.append(objorplugin)else:plugins.append(plugin)classLogCatcher(logging.Handler):"""Logging handler for collecting logged messages so you can check them later"""def__init__(self):self.messages=[]logging.Handler.__init__(self,logging.WARNING)defemit(self,record):self.messages.append(bb.build.logformatter.format(record))defcontains(self,message):return(messageinself.messages)