deleteDatasetSite.py

File contents

import os
import sys
import commands
import lfc
import dq2.info.ATLASCache
import dq2.info.TiersOfATLAS
from dq2.clientapi.DQ2 import DQ2
from dq2.filecatalog import create_file_catalog
from optparse import OptionParser

import errno
import logging
import logging.handlers


os.environ['LFC_CONNTIMEOUT']='1800'
os.environ['LCG_GFAL_VO']='atlas'
os.environ['LFC_CATALOG_TYPE']='lfc'


def main():
    usage = "%prog [-h/--help | options] <PATTERN>"
    parser = OptionParser(usage)
    parser.add_option("-d", "--dataset",help="Dataset name")
    parser.add_option("-l", "--logFile",help="Log file name")
    parser.add_option("-s", "--site",help="Site name")

    (options, args) = parser.parse_args()

    if options.dataset == None:
        print "Must specify dataset name.  Quitting..."
        sys.exit(1)

    if options.logFile == None:
        print "Using default log file /tmp/log_deletedataset.log"
        options.logFile="/tmp/log_deletedataset.log"

    if options.site == None:
        print "Must specify the site name.  Quitting..."
        sys.exit(1)

    rmcommand="srmrm -2"
    rmdircommand="srmrmdir -2"
    #dsn="data09_900GeV.00142166.physics_MinBias.merge.AOD.f196_m325"
    dsn=options.dataset
    siteName=options.site
    #siteName="DUKE_LOCALGROUPDISK"
    #siteName="BNL-OSG2_DATADISK"
    #logFileName="/home/hiroito/testrmlog.log"
    logFileName=options.logFile
    
    logger=logging.getLogger("logger")
    logger.setLevel(logging.DEBUG)
    handler=logging.handlers.RotatingFileHandler(logFileName,maxBytes=268435456,backupCount=10)
    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
    handler.setFormatter(formatter)

    logger.addHandler(handler)
        
    dq2client=DQ2()

    logger.info("Delete %s from %s" % (dsn, siteName))  
    #logger.nfo("Chaning replica archive bit to be 0") 
    #dq2client.setReplicaMetaDataAttribute(dsn, siteName, "archived", 0)
       
    fileList=dq2client.listFilesInDataset(dsn)

    logger.info("Deleting replica from the central replica catalog")
    dq2client.deleteDatasetReplicas(dsn, [siteName])

    dq2lfc = dq2.info.TiersOfATLAS.getLocalCatalog(siteName)
    lfcHost=dq2lfc[dq2lfc.find('://')+3:].split(":")[0]
    #print lfcHost
    sePort=":" + dq2.info.TiersOfATLAS.ToACache.sites[siteName]['srm'].split(":")[4].split("/")[0]
    #print sePort
    seService=""

    for i in dq2.info.TiersOfATLAS.ToACache.sites[siteName]['srm'].split(":")[4].split("?SFN")[0].split("/")[1:]:
        seService=seService + "/" + i

    #print seService

    seHost=dq2.info.TiersOfATLAS.ToACache.sites[siteName]['srm'].split(":")[3].strip("/")

    catCheck={}

    maxBulkSrmRm=10

    dirList=[]
    
    if len(fileList)>0:
        fileList=fileList[0]

        for guid in fileList:
            #print guid
            catCheck[guid]=fileList[guid]['lfn']

        catalog=create_file_catalog(dq2lfc)
        catalog.connect()
    
        response2 = catalog.bulkFindReplicas(catCheck)
        response = catalog.filterSiteReplicas(response2,[siteName])
        catalog.disconnect()

        #print response

        counterRm=0
        srmRmFiles=""
        #print "YES"
        s=lfc.lfc_startsess(lfcHost,"")
        if s!=0:
            logger.error("Could not connect to %s for deletion" % lfcHost)
        
        for guid in response:
            for i in response[guid]['surls']:
                counterRm=counterRm+1
                logger.info("Deleting %s from LFC %s" % (i, lfcHost))
                s=lfc.lfc_delreplica(guid,None,i)
                if s!=0:
                    if lfc.cvar.serrno!=errno.ENOENT:
                        logger.error("Failed to delete %s from LFC " % i)
                    
                if sePort not in i:
                    i=i.replace(seHost,seHost+sePort)

                if seService not in i:
                    i=i.replace(seHost+sePort, seHost+sePort+seService+"?SFN=")    
                
                srmRmFiles=srmRmFiles + " " + i

                fileDir=os.path.dirname(i)

                idx=-1
                try:
                    idx=dirList.index(fileDir)
                except:
                    dirList.append(fileDir)
                    
                           
                if counterRm%maxBulkSrmRm==0:
                    srmRmFiles= rmcommand + srmRmFiles
                    logger.info("Removing files from SRM by %s: " % srmRmFiles)
                    s, o = commands.getstatusoutput(srmRmFiles)
                    if s!=0:
                        logger.error("Failed to remove files: %s" % o)
                    print srmRmFiles
                    srmRmFiles=""

        s=lfc.lfc_endsess()
    
        if srmRmFiles != "":
            srmRmFiles= rmcommand + srmRmFiles
            logger.info("Remvoing files from SRM by %s " % srmRmFiles)
            s, o = commands.getstatusoutput(srmRmFiles)
            if s!=0:
                logger.error("Failed to remove files: %s" % o)
            print srmRmFiles

        for dirpath in dirList:
            srmRmDir=rmdircommand + " " + dirpath
            logger.info("Removing dirctory from SRM by %s " % srmRmDir)
            s, o = commands.getstatusoutput(srmRmDir)
            if s!=0:
                logger.error("Failed to remove directory: %s" % o)

            print srmRmDir    
            

            
if __name__ == "__main__":
    main()