#! /usr/bin/env python

#############################################################
#                                                           #
#   Author: Bertrand Neron , Herve Menager,                 #
#           Sandrine Larroude                               #
#   Organization:'Biological Software and Databases' Group, #
#                Institut Pasteur, Paris.                   #
#   Distributed under GPLv2 Licence. Please refer to the    #
#   COPYING.LIB document.                                   #
#                                                           #
#############################################################

"""
mobDeploy.py

This script is used to update the list of deployed and imported programs
on the mobyle server, as well as the various index files.
"""
import sys, os

MOBYLEHOME = None
if os.environ.has_key('MOBYLEHOME'):
    MOBYLEHOME = os.environ['MOBYLEHOME']
if not MOBYLEHOME:
    sys.exit('MOBYLEHOME must be defined in your environment')

if ( MOBYLEHOME ) not in sys.path:
    sys.path.append( MOBYLEHOME )
if ( os.path.join( MOBYLEHOME , 'Src' ) ) not in sys.path:    
    sys.path.append( os.path.join( MOBYLEHOME , 'Src' ) )

import logging
import shutil
import glob
from Ft.Xml.Domlette import NoExtDtdReader, Print
from Ft.Lib import UriException
import libxml2
import urllib2

from Mobyle import MobyleLogger
MobyleLogger.MLogger()
r_log = logging.getLogger('mobyle.registry' )
console = logging.StreamHandler(sys.stderr)
#r_log.setLevel(logging.INFO)

    
formatter = logging.Formatter('%(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
r_log.addHandler(console)

from Mobyle.Registry import Registry, registry
from Mobyle.Validator import Validator
from Mobyle.SearchIndex import SearchIndex
from Mobyle.ClassificationIndex import ClassificationIndex
from Mobyle.DataInputsIndex import DataInputsIndex
from Mobyle.DescriptionsIndex import DescriptionsIndex


import Mobyle.ConfigManager

class ProgramDeployer:
    """
    @author: Bertrand Neron
    @organization: Institut Pasteur
    @contact:mobyle@pasteur.fr
    """

    def __init__( self ):
        self.cfg = Mobyle.ConfigManager.Config()
        self.customPrefix = os.path.join ( self.cfg.mobylehome() , "Local" , "Programs")
        self.publicPrefix = os.path.join ( self.cfg.mobylehome() , "Programs")
        self.finalPrefix = self.cfg.programs_path()
        self.finalImportPrefix = self.cfg.programs_imports_path()
        self.tmpPrefix = registry.tmpPrefix
               
        
    def clean( self , program = 'all' , server = 'all'):
        """
        clean/remove a specific program or every program on every server
        @param program: program name
        @type program: string
        @param server: server name
        @type server: string
        """
        r_log.info( "Published programs: cleaning %s/%s" % (server, program) )
        if program == 'all':
            if server == 'local':
                directories = [self.finalPrefix]
            elif server == 'all':
                directories = [self.finalPrefix, self.finalImportPrefix]
            else:
                directories = [self.finalImportPrefix]
            programPaths = []
            for directory in directories:
                programPaths.extend(glob.glob( os.path.join( directory , '*.xml' ) ))
        else:
            programPaths = [registry.getProgramPath(program, server)]
        for path in programPaths:
            r_log.info("Published programs: removing %s" % path)
            try:
                os.unlink(path)
            except OSError , err:
                msg = "Program remove failed for path %s : %s" % (path, err)
                r_log.error( msg )

    
    def build(self , program = 'all'  , server = 'all' , force = False ):
        """
        predeploy or import a specific program or every program from every server
        @param program: program name
        @type program: string
        @param server: server name
        @type server: string
        """
                            
        if server == 'all':
            if program == 'all':
                 self._localDeploy()
                 self._remoteDeploy()
            else:
                self._localDeploy( program = program )
                self._remoteDeploy( program = program )
        elif server == 'local':
            if program == 'all':
                self._localDeploy()
            else:
                self._localDeploy( program , force = force )
        else:
            if program == 'all':
                self._remoteDeploy( server = server )
            else:
                self._remoteDeploy( server = server , program = program ) 
                           
    
    def index(self):
        """
        generate index files corresponding to the Deployed program
        """
        r_log.info("Regenerating indexes:")
        r_log.info("Reloading the registry...")
        registry.load(fromTmpDir=True)
        r_log.info("Regenerating search index...")
        SearchIndex.generate()
        r_log.info("Regenerating classification index...")
        ClassificationIndex.generate()
        r_log.info("Regenerating data inputs index...")
        DataInputsIndex.generate()
        r_log.info("Regenerating descriptions index...")
        DescriptionsIndex.generate()
        pass
    
    
    def tempArchi(self, program , server):
        """
        creation of a temporary architecture for a pre-deployment of the programs
        """
        dst = self.tmpPrefix
        src = self.finalPrefix
        src_imp = self.finalImportPrefix
        dst_imp = os.path.join(dst,"imports")
        dst_index = os.path.join(dst,"index")
        
        if os.access(dst, os.F_OK):
            shutil.rmtree(dst)

        #Implementation of the necessary architecture
        if server == 'local':
            if program == ['all']:
                os.mkdir(dst)
                shutil.copytree(src_imp, dst_imp)
            else:
                shutil.copytree(src, dst)
        else:
            if program == ['all']:
                self.clean( server = server )
            shutil.copytree(src, dst)                

        if not os.path.exists(dst):
            os.mkdir(dst)
        if not os.access(dst_imp, os.F_OK):
            os.mkdir(dst_imp)
        
        if os.path.exists(dst_index):
            shutil.rmtree(dst_index)
        os.mkdir(dst_index)
        
        
    def rmvArchi(self):
        """
        Clean temporary architecture and move our predeployed programs to their right directory
        """
        dst = self.finalPrefix
        try:
            shutil.rmtree(dst+"_old")
        except OSError:
            pass
        os.rename(dst, dst+"_old")
        os.rename(self.tmpPrefix, dst)
        shutil.rmtree(dst+"_old")
        

    def _deployXmlFile(self,srcPath,dstPath):
        ctxt = libxml2.createFileParserCtxt(srcPath)
        ctxt.replaceEntities(1)
        ctxt.parseDocument()
        doc = ctxt.doc()
        doc.xincludeProcess()
        dstFile = open(dstPath,"w")
        doc.saveTo(dstFile)
        dstFile.close()
    
    def _validateOrDie(self,path, publicURI):
        """
        checks if the program on the specified path validates.
        logs validation errors
        if it does not validate, the file is removed
        Warning: validation should be done in the target path, to avoid file loss.
        @param path: path to the program file
        @type path: string
        @return: True if it validates, False otherwise
        @rtype: list of strings
        """
        try:
            val = Validator(path, publicURI)
            val.run()
            if not(val.valOk):
               r_log.error("Testing : Deployment of %s ABORTED, it does NOT validate. If it exists, the previous xml version is temporary kept. Details follow:" % str(path))
               if val.runRNG and val.rngOk==False:
                   r_log.error("* relax ng validation failed - errors detail:")
                   for re in val.rngErrors:
                       r_log.error(" - %s" % re)
               if val.runSCH and len(val.schErrors)>0:
                   r_log.error("* schematron validation failed - errors detail:")
                   for se in val.schErrors:
                       r_log.error(" - %s" % se)       
               return False            
            return True
        except Exception, e:
            r_log.error("Testing : Deployment of %s ABORTED (if it exists, the previous xml version is temporary kept). An unexpected error happened during its validation. %s" % (str(path), e))
            return False    
        
        
        
    def _keepPreviousXml(self, previousXml, destXml, c = True):
        """
        In case of problem with a xml to deploy, keep the previous version if it exists
        @param previousXml: hypothetic previous Xml path
        @type previousXml: string
        @param destXml: current Xml-to-deploy path
        @type destXml: string
        @param n: boolean to tell if the xml has to be deleted or not
        @param n: boolean 
        """
        if c:
            os.unlink(destXml)
            
        if os.access(previousXml, os.F_OK):
            shutil.copyfile(previousXml, destXml)
            
        
    ###################################################
    #                                                 #
    #   private methods for local server operations   #
    #                                                 #
    ###################################################
    
    
    def _localDeploy(self , program = 'all' , force = False ):
        """
        PreDeploy one or all of the valid local programs
        @param program: program name
        @type program: string
        """
        
        validPrograms = []
        
        r_log.info( "Testing validity and Deploying local program(s): %s/%s" % (server, program))
        if program == 'all' :
            programs = self._localPrograms2deploy()
        else:
            if not self._isInConfig( program ):
                if force :
                    r_log.info( "Force pre-deploying local program: %s" %program )
                else:
                    r_log.info( "The program you want to deploy is not in Config. Add it in config or use force option to deploy it" )
                    sys.exit( 0 )
                
            customPath = os.path.join(self.customPrefix, program + '.xml')
            if os.path.exists( customPath ):
                programs = [ customPath ]
            else:                
                publicPath = os.path.join(self.publicPrefix, program + '.xml')
                if os.path.exists( publicPath ):
                    programs = [ publicPath ]
                else:
                    r_log.error( "Local program deploy failed, path not found: %s" % program)
                    sys.exit(1)
                    
        #Pre deploy valid program(s)
        for path in programs:
            destPath = os.path.join(self.tmpPrefix, os.path.basename( path ))
            try:
                self._deployXmlFile(path,destPath)
                os.chmod(destPath, 0644)  
            except Exception , err:
                msg = "Testing : Program deploy failed for path %s : %s." % ( path , err )
                r_log.error( msg ) 
            
            if self._validateOrDie(destPath, None):
                validPrograms.append(destPath)
            # if the xml is not valid and a previous version exists, we keep the previous version 
            else:
                previous_xml = os.path.join(self.finalPrefix, os.path.basename( path ))
                self._keepPreviousXml(previous_xml, destPath)

        #Print the sorted list of valid programs pre-deployed
        validPrograms = sorted(validPrograms)
        for path in validPrograms:
            d = os.path.join(self.tmpPrefix, os.path.basename( path ))
            r_log.info("Pre-deploying local program: %s" % os.path.basename( path ))
                             

    def _uniq(self , publicPath , customPath ):
        """
        @return: a list containing one xml path per service with the priority to the custom xml   
        """
        result = {}
        for path in publicPath:
                result[ os.path.basename( path )[:-4] ] = path
        for path in customPath:
            result[ os.path.basename( path )[:-4] ] = path
        return result.values()
    

    def _localPrograms2deploy(self ):
        """
        @return: the list of local (from this server) program file paths to deploy
        @rtype: list of strings
        """
        programs = {}
        for method in self.cfg.programs_deployment_order():
            getattr( self, '_' + method )( programs )
        
        return programs.values()
    
    
    def _isInConfig( self , programName ):
        """
        @param programName: the name of a program
        @type programName: string
        @return: True if the programName is in the program define in Config.py, False otherwise
        @rtype: boolean
        """
        programs = {}
        for method in self.cfg.programs_deployment_order():
            getattr( self, '_' + method )( programs )
         
        if programs.has_key( programName ):
            return True
        else:
            return False
        
        
    def _include( self , programs ):
        """
        @return: the list of local (from this server) program file paths include in deploy
        @rtype: list of strings
        """
        for mask in self.cfg.programs_deployment_include():
            for path in self._uniq( glob.glob( os.path.join( self.publicPrefix , mask + '.xml' )) , 
                                    glob.glob( os.path.join( self.customPrefix , mask + '.xml' ))
                                    ):
                programs[ os.path.basename( path )[:-4] ] = path
            
        return programs

        
    def _exclude( self , programs ):
        """
        @return: the list of local (from this server) program file paths to exclude from deploy
        @rtype: list of strings
        """
        for mask in self.cfg.programs_deployment_exclude():
            for path in self._uniq( glob.glob( os.path.join( self.publicPrefix , mask + '.xml' )) ,
                                    glob.glob( os.path.join( self.customPrefix , mask + '.xml' ))
                                    ):
                try:
                    del( programs[ os.path.basename( path )[:-4] ] )
                except KeyError:
                    pass
        return programs    
    
    
    ###################################################
    #                                                 #
    #  private methods for remote server operations   #
    #                                                 #
    ###################################################

    def _remoteDeploy(self, server='all', program = 'all'):
        """
        deploy one or all of the remote valid programs
        @param server: program name
        @type server: string
        @param program: program name
        @type program: string
        """
        
        validPrograms = {}
        
        r_log.info( "Testing and Deploying imported program(s): %s/%s" % (server, program))
        if server=='all':
            remoteServers = registry.serversByName
            del remoteServers['local']
            serverNames = remoteServers.keys()
        else:
            serverNames = [server]
        if program == 'all' :
            programs = [(p.server.name, p.name, p.url) for p in registry.programs if p.server.name in serverNames]
        else:
            programs = [(p.server.name, p.name, p.url) for p in registry.programs if p.name==program and p.server.name in serverNames]
        
        #List of valid program(s)
        for server, program, url in programs:
            p = registry.getProgramPath(program, server) 
            tmpDest = os.path.join(self.tmpPrefix, "imports" , os.path.basename(p))
            #test if the url is accessible
            try:
                url_ok = urllib2.urlopen(url)
                try:
                    self._deployXmlFile(url,tmpDest)
                    os.chmod( tmpDest , 0644 )
                    if not self._validateOrDie(tmpDest, url):
                        self._keepPreviousXml(os.path.join(self.finalImportPrefix, os.path.basename(p)), tmpDest)
                    else:
                        validPrograms[tmpDest]=url
                except  Exception, err:
                    msg = "Remote program deploy failed for path %s, if it exists, the previous xml version is temporary kept. Details follow : %s" % (url, err)
                    r_log.error( msg )
                    self._keepPreviousXml(os.path.join(self.finalImportPrefix, os.path.basename(p)), tmpDest, False)
            except  Exception, err:
                msg = "Remote program %s is not accessible, deployment ABORTED; if it exists, the previous xml version is temporary kept. Details follow : %s" % (url, err)
                r_log.error( msg )
                self._keepPreviousXml(os.path.join(self.finalImportPrefix, os.path.basename(p)), tmpDest, False)

        #Print the sorted list of valid programs pre-deployed       
        for path in validPrograms.keys():
            d = os.path.join(self.tmpPrefix,'imports', os.path.basename( path ))
            r_log.info("Pre-deploying remote program: %s from %s" % (os.path.basename( path), validPrograms[path] ))  



if __name__ == '__main__':
    import getopt

    def usage():
        return """usage mobdeploy <options> command
    commands
           deploy   : deploy program in    available options [-p , -s ]
           clean   : clean the repository  available options [-p , -s ]
           index   : generate indexes      no options
    options
           -p --programs programName : apply the command only on these programs. 
               The programs must be separated by commas (without spaces).
           -s --server MobyleServerName : apply the command on programs from 
                this server. default is 
                    'all' if -p is NOT specified 
                    'local' if -p is specified
           -f --force : force to deploy a program even it is not in the config.py
           -h --help : print this
           -d --detailled : Maximum log level with all information. By default, only errors are printed.
           
    examples:
           mobdeploy.py deploy 
               deploy all programs defined in configuration ( local and remote )
           
           mobdeploy.py  --program squizz deploy
               deploy local squizz interface only
           
           mobdeploy.py  --program squizz --force deploy
               deploy local squizz interface only even squizz is not in config.py
           
           mobdeploy.py --server marygay deploy
               deploy all programs imported from Mobyle server named "marygay"
                       
           mobdeploy clean
               clean program repository from all interfaces
                
           mobdeploy -p squizz clean
               remove only squizz interface
               
           mobdeploy index
           
    """   

    try:
        opts, cmds = getopt.getopt(sys.argv[1:], "hfdp:s:", ["help", "force" , "detailled", "programs=", "server="])
    except getopt.GetoptError , err :
        r_log.error(err)
        print >> sys.stderr, usage()
        sys.exit( 1 )
    
    if len( cmds ) != 1 :
        print >> sys.stderr, usage()
        sys.exit( 1 )
    else:
        command = cmds[0]
        deployer = ProgramDeployer()
        
        server = 'all'
        programs = [ 'all' ]
        server = None
        force, detail = False, False
        for opt , value in opts:
            if opt == "-s" or opt == "--server" :
                server = value
            elif opt == "-p" or opt == "--programs":
                programs = value.split( ',')
            elif opt == "-f" or opt == "--force":
                force = True
            elif opt == "-d" or opt == "--detailled":
                detail = True  
            if opt == "-h" or opt == "--help":
                print >> sys.stderr, usage()
                sys.exit( 0 )

        if server == None:
            if programs !=[ 'all' ]: 
                server = 'local'
                r_log.info("*******Be careful, no server precised. By default, server = local.*******")
            else:
                r_log.info("*******Be careful, no server precised. By default, server = all.*******")
                server = 'all'
        elif programs != [ 'all' ] and server == 'all':
            print >> sys.stderr , "When program is specified, server must be specified (\"all\" is not allowed )"
            print >> sys.stderr, usage()
            sys.exit( 1 )
        
        if detail:
            r_log.setLevel(logging.INFO)
        else:
            r_log.setLevel(logging.ERROR)

        if command == "deploy":
            deployer.tempArchi(program = programs , server = server)
            for program in programs:
                if program:
                    deployer.build( program = program , server = server , force = force )
            deployer.index()
            deployer.rmvArchi()
            r_log.info("Program(s) now published.")
        elif command  == "clean":
            for program in programs:
                if program:
                    deployer.clean( program = program , server = server )
            deployer.tempArchi(program = None, server = None)
            deployer.index()
            deployer.rmvArchi()
        elif command == "index":
            if  opts:
                print >> sys.stderr, usage()
                sys.exit( 1 )
            else:
                deployer.tempArchi(program = None, server = None)
                deployer.index()
                deployer.rmvArchi()
        else:
            print >> sys.stderr, usage()
            sys.exit( 1 )
        sys.exit( 0 )
