532""" Simple test - anything before Aug 2012 is redone. 533 """ 534# @FIXME If old deeps contain deprecated inputs, do not include 535# Check if any Required X has multiple OBs - deeps needed and 536# latest deep is at position... 537deepStacks=set(pIDforpIDinself.reqProdsOfType['stack'] 538ifself.imageProductDict[pID]) 539 540# Get ones that are deepStacks and not goodOldDeepPids. 541goodOldDeepPids=set(self.archive.query( 542"distinct s.productID", 543"Multiframe as m,ProgrammeFrame as p,RequiredStack as s", 544"p.programmeID=%s and p.productID>0 and p.multiframeID=m.multiframeID " 545"and frameType like 'deep%%stack' and frameType not like " 546"'%%tile%%' and deprecated in (0,255) and creationDate " 547"BETWEEN '20120831' AND '21000101' and s.filterID=m.filterID and " 548"s.programmeID=p.programmeID and " 549"dbo.fGreatCircleDist(s.ra,s.dec,(15.*rabase),decbase)<(60.*s.stackRadius)" 550%self.programmeID)) 551 552potDodgyDeeps=deepStacks-goodOldDeepPids 553 554# Are any of these oldDeepsPids 555# Could just be new pointings 556# @FIXME: Sometimes components in old deeps deprecated and now just a 557# single pointing...... 558# Optional is it safe to continue... instead of error, listing 559# PIDs which may have problems... 560oldDeepsPids=self.archive.query( 561"distinct s.productID", 562"Multiframe as m,ProgrammeFrame as p,RequiredStack as s", 563"p.programmeID=%s and p.productID>0 and p.multiframeID=m.multiframeID " 564"and frameType like 'deep%%stack' and frameType not like " 565"'%%tile%%' and deprecated in (0,255) and s.filterID=m.filterID and " 566"s.programmeID=p.programmeID and " 567"dbo.fGreatCircleDist(s.ra,s.dec,(15.*rabase),decbase)<(60.*s.stackRadius) " 568"and creationDate BETWEEN '20090930' AND '20120831'" 569%self.programmeID) 570 571returnpotDodgyDeeps.intersection(oldDeepsPids)

1037"""1038 Extracts the catalogue for the given product, using the extraction1039 tool for the programme.10401041 """1042# @TODO: Is this OBSOLETE. Not quite yet. Needs rewriting1043Logger.addMessage("Extracting catalogue for %s %s"1044%(bestProduct.productType,bestProduct.fileName()),1045alwaysLog=(bestProduct.productType=='mosaic'))10461047#isNewCat = not self.archive.isTrialRun and not bestProduct.catExists()\1048# and (self.extractor == "CASU" or self.extractor == "SEX")1049#@TODO: Revert to above1050#isNewCat = not bestProduct.catExists() \1051# and (self.extractor == "CASU" or self.extractor == "SEX")1052productID=bestProduct.deepProductTable["productID"][bestProduct.index]1053prodOfID={productID:bestProduct}1054componentsOfProd={productID:components}1055# @TODO: use ProductProcessing to determine what is to be run1056ifself.extractor=="CASU":1057processLayers=self.calcProcessLayers('imcore',[productID],prodOfID,None)1058forproductInfo,_isPartsinprocessLayers:1059productIDs=[prodIDforprodID,_partNuminproductInfo]1060self.runInitialCasuExtraction(productIDs,prodOfID,1061componentsOfProd,stackInfoDict=None)1062elifself.extractor=="SEX":1063# Only use DUAL FILTER if mosaic1064processLayers=self.calcProcessLayers('sex',[productID],prodOfID,None)1065forproductInfo,_isPartsinprocessLayers:1066productIDs=[prodIDforprodID,_partNuminproductInfo]1067ifproductIDs:1068sx=SExtractorInterface(self.archive,bestProduct,dualImageProduct)1069sx.extractCatalogue()1070sx.calcAperCor()1071self.productProcessing(productIDs,prodOfID,'sex')1072processLayers=self.calcProcessLayers('calcZP',[productID],prodOfID,1073componentsOfProd)1074forproductInfo,_isPartsinprocessLayers:1075productIDs=[prodIDforprodID,_partNuminproductInfo]1076ifproductIDs:1077Logger.addMessage("Calculating zero-point and updating headers...",1078alwaysLog=(bestProduct.productType=='mosaic'))10791080fits.removeDuplicateKeywords(bestProduct.catName())10811082# Update FITS headers1083ifnotisQuick:1084updateHeaders(bestProduct,self.calcZP(bestProduct,components),1085pu.Provenance(self.archive,bestProduct.fileName(),1086components=components))1087self.productProcessing([productID],prodOfID,'calcZP')1088self.testProduct(bestProduct,components)1089self.productProcessing([productID],prodOfID,'testProduct')1090# Do not compress...1091self.productProcessing([productID],prodOfID,isComplete=True)

1246"""1247 """1248ifbestProduct.isTile()orbestProduct.isMosaic():1249returnallFiles1250deeps=[afforafinallFilesifself.sysc.deepSuffixinaf.image]1251nonDeeps=[afforafinallFilesifself.sysc.deepSuffixnotinaf.image]1252whiledeeps:1253# Should only be one deep1254iflen(deeps)>1:1255Cu13.CuError("More than one old deep used to construct %s"%bestProduct.fileName())1256# Get OB files..1257provFileNames=self.archive.query(1258"m2.fileName","Multiframe as m1,Provenance as v,Multiframe as m2",1259"v.combiframeID=m1.multiframeID and m1.fileName like '%%%s' and "1260"v.multiframeID=m2.multiframeID order by m2.mjdObs"%1261os.path.join(os.path.basename(os.path.dirname(deeps[0].image)),1262os.path.basename(deeps[0].image)))1263newFiles=[fits.FileList(pfn,fits.getConfMap(pfn),fits.getCatalogue(pfn))forpfninprovFileNames]1264nonDeeps=newFiles+nonDeeps1265deeps=[afforafinnewFilesifself.sysc.deepSuffixinaf.image]1266returnnonDeeps

1825"""1826 """1827components=bestProduct.findRequiredInputs()1828ifbestProduct.isTile():1829# Check to see if data in GroutingLinks1830prodID=bestProduct.deepProductTable["productID"][bestProduct.index]1831ifself.archive.queryEntriesExist(1832"GroutingLinks",1833"programmeID=%s AND tileName='%s' AND productID=%s"1834%(self.programmeID,bestProduct.fileName(),prodID)):1835linkList=[(provFile,os.path.join(tmpGroutDir,linkName))1836forprovFile,linkNameinself.archive.query(1837"origProvFile,linkName",1838"GroutingLinks",1839"programmeID=%s AND tileName='%s' AND productID=%s"1840%(self.programmeID,bestProduct.fileName(),prodID))]1841Logger.addMessage("Selecting link list from archive")1842linkOutput=[]1843else:1844linkOutput=[]1845linkList=[]1846forfilePath,mfID,fType \ 1847inpu.getWholeProv(self.archive,bestProduct.fileName(),1848components):1849ifself.sysc.filtPrefixinos.path.basename(filePath):1850origFilePath=os.path.join(os.path.dirname(filePath),1851os.path.basename(filePath)1852.replace(self.sysc.filtPrefix,''))18531854linkFileList=[filePath]18551856else:1857origFilePath=filePath1858linkFileList=[]1859# These are not correct for normals1860ifnotfType=='normal':1861provConfName=fits.getConfMap(origFilePath)1862provCatName=fits.getCatalogue(origFilePath)1863linkFileList+=[origFilePath,provConfName,provCatName]1864else:1865provConfName=self.archive.query(1866selectStr="c.fileName",1867fromStr="Multiframe as c,Multiframe as m",1868whereStr="c.multiframeID=m.confID and m.multiframeID=%s"1869%mfID,firstOnly=True)1870linkFileList+=[origFilePath,provConfName]1871forfilePinlinkFileList:1872linkName=os.path.basename(fileP)1873ifself.sysc.stackSuffixinfileP:1874linkName=os.path.join(1875os.path.basename(os.path.dirname(fileP)),linkName)1876linkOutput.append((self.programmeID,prodID,1877bestProduct.fileName(),fileP,linkName))1878# insert into table1879linkName=os.path.join(tmpGroutDir,linkName)1880linkList.append((fileP,linkName))1881linkList=list(set(linkList))1882linkOutput=list(set(linkOutput))1883forfileP,linkNameinlinkList:1884ifnotos.path.exists(linkName):18851886Logger.addMessage("Creating soft link from %s to %s"1887%(fileP,linkName),alwaysLog=False)18881889utils.ensureDirExist(os.path.dirname(linkName))1890extp.run('ln -s %s %s'%(fileP,linkName),1891isVerbose=False)1892forloinlinkOutput:1893self.archive.insertData("GroutingLinks",lo,enforcePKC=True)

1934"""1935 Sets up attributes to convert an external mosaic into the1936 correct format: MEF, with the correct headers for CU3 to ingest1937 into the science archive.19381939 """1940ifnot(componentsandos.path.exists(bestProduct.provFile())):1941raiseCu13.CuError("External mosaics have not been imported or "1942"set up correctly. If they have been imported "1943"add a line into ExternalProduct and run "1944"cu13.convertProvInfo.py")19451946bestProduct.imageName=components[0][1]1947bestProduct.weightName=components[0][2]1948prodID=bestProduct.deepProductTable["productID"][bestProduct.index]1949Logger.addMessage("Converting mosaic productID %s in %s"1950%(prodID,1951','.join(map(str,bestProduct.deepProductTable["productID"]))))1952195319541955# convert into mef if necessary and sort out headers1956outputFiles=[(bestProduct.fileName(),True),1957(bestProduct.confName(),False)]19581959# normalise weight image1960normalFactor=(1ifself.extractor!='CASU'else1961getNormalisationConst(bestProduct))19621963forimageName,isImageinoutputFiles:1964self.convertMosaic(bestProduct,imageName,isImage,provInfoObj,1965normalFactor)1966196719681969Logger.addMessage("Finished converting mosaic productID %s"1970%(prodID))

1976"""1977 Converts a mosaic to MEF and adds the correct primary and extension1978 header.19791980 """1981ifnot(os.path.exists(outputFileName)):1982Logger.addMessage("Converting mosaic %s %s..."%1983("image"ifisImageelse"confidence map",outputFileName))1984# @TODO: Check to see if this part has already been done.1985# use ProductProcessing1986inputFileName=mosaicObj.imageNameifisImage \ 1987elsemosaicObj.weightName1988# Copy image and get header info1989ifnotmosaicObj.isMef(inputFileName):1990ifnotself.archive.isTrialRun:1991ifnotself.redoHeader:1992origImagHeader=[convertSFtoMEF(inputFileName,1993outputFileName,isImage,self.extractor,normalFactor)]19941995os.chmod(outputFileName,0644)1996else:1997origImagHeader=[fits.open(inputFileName)[0].header]1998else:1999ifnotself.archive.isTrialRun:2000ifnotself.redoHeader:2001shutil.copyfile(inputFileName,outputFileName)2002hdulist=fits.open(outputFileName,'update')2003origImagHeader=[hdulist[0].header,hdulist[1].header]2004ifnotisImageandself.extractor=='CASU':2005hdulist[1].data=numpy.multiply(2006hdulist[1].data,normalFactor).astype(2007numpy.uint8)2008fits.checkForUnicodeError(hdulist,outputFileName)2009hdulist.close()2010else:2011hdulist=fits.open(inputFileName)2012origImagHeader=[hdulist[0].header,hdulist[1].header]2013hdulist.close()20142015# Now fix up headers2016ifnotself.archive.isTrialRun:2017hdulist=fits.open(outputFileName,'update')2018hdulist[0].header=self.getPrimaryHeader(mosaicObj,2019hdulist[0].header,origImagHeader[0],isImage,provObj)20202021fits.checkForUnicodeError(hdulist,outputFileName)2022ifmosaicObj.isMef():2023hdulist[1].header=self.getInitImageHeader(2024hdulist[1].header,origImagHeader[len(origImagHeader)-1],2025provObj)20262027fits.checkForUnicodeError(hdulist,outputFileName)2028hdulist[1].header=self.getImageHeader(mosaicObj,2029hdulist[1].header,isImage,provObj)2030fits.checkForUnicodeError(hdulist,outputFileName)2031hdulist.close()