deleteDatasetSite.py
deleteDatasetSite.py
—
text/python-source,
5 kB (5425 bytes)
File contents
import os import sys import commands import lfc import dq2.info.ATLASCache import dq2.info.TiersOfATLAS from dq2.clientapi.DQ2 import DQ2 from dq2.filecatalog import create_file_catalog from optparse import OptionParser import errno import logging import logging.handlers os.environ['LFC_CONNTIMEOUT']='1800' os.environ['LCG_GFAL_VO']='atlas' os.environ['LFC_CATALOG_TYPE']='lfc' def main(): usage = "%prog [-h/--help | options] <PATTERN>" parser = OptionParser(usage) parser.add_option("-d", "--dataset",help="Dataset name") parser.add_option("-l", "--logFile",help="Log file name") parser.add_option("-s", "--site",help="Site name") (options, args) = parser.parse_args() if options.dataset == None: print "Must specify dataset name. Quitting..." sys.exit(1) if options.logFile == None: print "Using default log file /tmp/log_deletedataset.log" options.logFile="/tmp/log_deletedataset.log" if options.site == None: print "Must specify the site name. Quitting..." sys.exit(1) rmcommand="srmrm -2" rmdircommand="srmrmdir -2" #dsn="data09_900GeV.00142166.physics_MinBias.merge.AOD.f196_m325" dsn=options.dataset siteName=options.site #siteName="DUKE_LOCALGROUPDISK" #siteName="BNL-OSG2_DATADISK" #logFileName="/home/hiroito/testrmlog.log" logFileName=options.logFile logger=logging.getLogger("logger") logger.setLevel(logging.DEBUG) handler=logging.handlers.RotatingFileHandler(logFileName,maxBytes=268435456,backupCount=10) formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) dq2client=DQ2() logger.info("Delete %s from %s" % (dsn, siteName)) #logger.nfo("Chaning replica archive bit to be 0") #dq2client.setReplicaMetaDataAttribute(dsn, siteName, "archived", 0) fileList=dq2client.listFilesInDataset(dsn) logger.info("Deleting replica from the central replica catalog") dq2client.deleteDatasetReplicas(dsn, [siteName]) dq2lfc = dq2.info.TiersOfATLAS.getLocalCatalog(siteName) lfcHost=dq2lfc[dq2lfc.find('://')+3:].split(":")[0] #print lfcHost sePort=":" + dq2.info.TiersOfATLAS.ToACache.sites[siteName]['srm'].split(":")[4].split("/")[0] #print sePort seService="" for i in dq2.info.TiersOfATLAS.ToACache.sites[siteName]['srm'].split(":")[4].split("?SFN")[0].split("/")[1:]: seService=seService + "/" + i #print seService seHost=dq2.info.TiersOfATLAS.ToACache.sites[siteName]['srm'].split(":")[3].strip("/") catCheck={} maxBulkSrmRm=10 dirList=[] if len(fileList)>0: fileList=fileList[0] for guid in fileList: #print guid catCheck[guid]=fileList[guid]['lfn'] catalog=create_file_catalog(dq2lfc) catalog.connect() response2 = catalog.bulkFindReplicas(catCheck) response = catalog.filterSiteReplicas(response2,[siteName]) catalog.disconnect() #print response counterRm=0 srmRmFiles="" #print "YES" s=lfc.lfc_startsess(lfcHost,"") if s!=0: logger.error("Could not connect to %s for deletion" % lfcHost) for guid in response: for i in response[guid]['surls']: counterRm=counterRm+1 logger.info("Deleting %s from LFC %s" % (i, lfcHost)) s=lfc.lfc_delreplica(guid,None,i) if s!=0: if lfc.cvar.serrno!=errno.ENOENT: logger.error("Failed to delete %s from LFC " % i) if sePort not in i: i=i.replace(seHost,seHost+sePort) if seService not in i: i=i.replace(seHost+sePort, seHost+sePort+seService+"?SFN=") srmRmFiles=srmRmFiles + " " + i fileDir=os.path.dirname(i) idx=-1 try: idx=dirList.index(fileDir) except: dirList.append(fileDir) if counterRm%maxBulkSrmRm==0: srmRmFiles= rmcommand + srmRmFiles logger.info("Removing files from SRM by %s: " % srmRmFiles) s, o = commands.getstatusoutput(srmRmFiles) if s!=0: logger.error("Failed to remove files: %s" % o) print srmRmFiles srmRmFiles="" s=lfc.lfc_endsess() if srmRmFiles != "": srmRmFiles= rmcommand + srmRmFiles logger.info("Remvoing files from SRM by %s " % srmRmFiles) s, o = commands.getstatusoutput(srmRmFiles) if s!=0: logger.error("Failed to remove files: %s" % o) print srmRmFiles for dirpath in dirList: srmRmDir=rmdircommand + " " + dirpath logger.info("Removing dirctory from SRM by %s " % srmRmDir) s, o = commands.getstatusoutput(srmRmDir) if s!=0: logger.error("Failed to remove directory: %s" % o) print srmRmDir if __name__ == "__main__": main()