#!/usr/bin/env python

import os
import string
import urllib
import simplejson

import mb_cgi

import re
from lxml import etree
from Mobyle.JobFacade import JobFacade
from Mobyle.Workflow import Workflow, Task, Parameter, Paragraph, Link, Type, Datatype, Biotype, InputValue, Value, VElem, Vlist,\
    Parser
from Mobyle.Parser import parseService
from Mobyle.DataInputsIndex import DataInputsIndex
from Mobyle.Classes.DataType import DataTypeFactory
from Mobyle.SearchIndex import SearchIndex
from Mobyle.Registry import registry
import Mobyle.Net
from Mobyle.MobyleError import SessionError, UserValueError, MobyleError

from Mobyle.AuthenticatedSession import AuthenticatedSession
from Mobyle.AnonymousSession import AnonymousSession
from Mobyle.JobState import url2path
from Mobyle.JobState import JobState, ProgramJobState

# Library import for application category classification
from Mobyle.Registry import CategoryDef, registry, ServiceTypeDef
from Mobyle.ClassificationIndex import ClassificationIndex
from Mobyle.InterfacePreprocessor import InterfacePreprocessor
from Mobyle.Utils import emailHelpRequest

from Mobyle.BMPSWorkflow import BMPSWorkflow,RenameError

# Import treenode module - used for creating classification data in tree format
import treenode



# This CGI script will be used for generic services for the Mobyle Workflow projects
# the form field 'action' will be used for deciding the functions performed. The possible
# values of 'action' field are 
# action=get_wf&wf_name=xxx
# action=upload_wf_graphml&graphml=xxx&wf_name=yyy
# action=rename_wf&wf_from_name=xxx&wf_to_name=yyy
# action=delete_wf&wf_name=xxxx
# action=graphviz_layout&wf_name=xxxx

# Vivek Gopalan 05JAN2010

graphml_file_name = ".workflow_template_graphml.xml"

def intersect(seq1, seq2):
    """ Utility method to fetch the common elements from 2 arrays."""
    res = []                     # start empty
    for x in seq1:               # scan seq1
        if x in seq2:            # common item?
            res.append(x)        # add to end
    return res

def get_classification(lines):
    """Utility method to fetch create Tree structure in JSON array from the classification lines of the Mobyle XML elements"""
    
    # Create the root node
    root_node = treenode.TreeNode("All apps");
    node_data = {}
    line_iter = iter(lines)
    # Iterate over the every lines
    while 1:
        try:
            line = line_iter.next()
            line = line.rstrip()
            fields = line.split(':') # Split based on ":" separator
            # Remove the program name in beginning of the array and add it to the end
            prog_name = fields.pop(0)
            fields.append(prog_name)
            no_of_fields = len(fields)
            # Parse through the fields starting from index 1
            for i in range(1, no_of_fields) :
                child = fields[i];
                parent = fields[ i - 1 ]
                # Unique identifier based on all the parents  
                id = ":".join(fields[0: i]);

                # If the classification is already there
                if node_data.has_key(id) :
                    node = node_data[id]
                # If the classification is not there
                # the create a new node 
                else :
                    node = treenode.TreeNode(parent)
                    node_data[id] = node
                    # Add the child node structure to the root
                    # This has to be done in the first iteration
                    if i == 1 :
                        root_node.add_child(node)
                # Add child to the node 
                if not node.has_child_with_name(child) :
                    child_node = treenode.TreeNode(child)
                    node.add_child(child_node)
                    id = ":".join(fields[0:i + 1])
                    node_data[id] = child_node
        except StopIteration:
            break
    return root_node

def get_pipeout_params(program_name, param_name,classification_list ):
    """
    Utility method to fetch the output applications and its parameters. This is based
    on the comparison of the superclass,class and biotypes values of all the applications 
    in Mobyle with that of the input application name and parameter.
    """
    out_param_list=[]
    # Fetch all the program and viewer data input indexes
    pdi = DataInputsIndex("program")
    vdi = DataInputsIndex("viewer")
    # Fetch the unique URL identifier of the program from the registry
    program_url = registry.serversByName['local'].programsByName[program_name].url
    # Fetch the input data type of the program
    service = parseService(program_url)
    inp_datatype = service.getDataType(param_name)
    inp_biotypes = service.getBioTypes(param_name)
    df = DataTypeFactory()
    # get all the data input options from the indexes
    dataInputsList = pdi.getList().values() + vdi.getList().values()
    
    # parse through every datatype of all the mobyle programs
    for parameter in dataInputsList:
        try:
            # 1. fetch the datatype and biotype
            if parameter.get('dataTypeSuperClass'):
                dt = df.newDataType(parameter.get('dataTypeSuperClass'), parameter.get('dataTypeClass'))
            else:
                dt = df.newDataType(parameter.get('dataTypeClass'))
            if dt.isMultiple():
                dt = dt.dataType
        except MobyleError, me:
            mb_cgi.c_log.warning(me)
            continue
        datatype_match = True
        biotype_match = True
        bio_types = parameter.get('biotypes')
        # 2. Match the data type and biotype of the mobyle program with the input values
        if (inp_biotypes is not None and bio_types is not None and len(intersect(inp_biotypes, bio_types)) == 0):
            biotype_match = False
        if inp_datatype.name not in dt.ancestors:
            datatype_match = False
        if (datatype_match and biotype_match):
            pattern = re.compile("%s:" % parameter.get('programName'))
            subs_val = parameter.get('programName') + '|' + parameter.get('name') + ":"
            c= [re.sub(pattern,subs_val,l.rstrip()) for  l in classification_list if (pattern.match(l.rstrip()) is not None)]
            # populate output list if there is datatype and biotype match
            if c:
                out_param_list.extend(c)
    return out_param_list

def get_pipeout_status(program_name,param_name,params_string):
    """ 
    Utility method to fetch the status of the output parameter of a Mobyle program name.
    The status is reported only to the subset of parameters specified in the params_string.
    
    Used to get the status of an program's parameters for connecting to the already existing
    programs in the workflow canvas.
    
    @param program_name: Mobyle program name
    @param param_name: Parameter of the program
    @param params_string: Comma-delimited program name:parameter string 
    """
    out_status_list=[]
    # Open the input data type indexes of all the programs in Mobyle 
    pdi = DataInputsIndex("program")
    vdi = DataInputsIndex("viewer")
    ## For testing
    #program_name = 'clustalw-multialign'
    #param_name = 'newtreefile'
    
    # Fetch the unique url and datatypes for the input parameter
    program_url = registry.serversByName['local'].programsByName[program_name].url
    service = parseService(program_url)
    inp_datatype = service.getDataType(param_name)
    inp_biotypes = service.getBioTypes(param_name)
    df = DataTypeFactory()
    dataInputsList = pdi.getList().values() + vdi.getList().values()
    
    params_list = params_string.split(",")
    status_hash ={}
    
    for parameter in dataInputsList:
        try:
            # filter the results only to the input parameters - skip the other program:parameter names
            if (parameter.get('programName') + ':' + parameter.get('name') not in params_list):
                continue
            # Fetch the datatype and biotypes of the mobyle parameter
            if parameter.get('dataTypeSuperClass'):
                dt = df.newDataType(parameter.get('dataTypeSuperClass'), parameter.get('dataTypeClass'))
            else:
                dt = df.newDataType(parameter.get('dataTypeClass'))
            if dt.isMultiple():
                dt = dt.dataType
            datatype_match = True
            biotype_match = True
            bio_types = parameter.get('biotypes')
            # Camparate the datatype and biotype with that of the input parameter
            if (inp_biotypes is not None and bio_types is not None and len(intersect(inp_biotypes, bio_types)) == 0) :
                bioTypes = False
            if inp_datatype.name not in dt.ancestors:
                datatype_match = False
            if (datatype_match and biotype_match):
                # Store the status if the datatype and biotype matches
                status_hash[parameter.get('programName') + ':' + parameter.get('name')] = True
        except:
            mb_cgi.c_log.error("Error computing pipes", exc_info=True)
    # Process the results as list
    for x in params_list:            
        if(status_hash.has_key(x)):
            out_status_list.append("True")
        else:
            out_status_list.append("False")
    return ",".join(out_status_list)

def generate_classification(node):
    """
    Utility method to generate tree structure for client-side processing from the Tree object
    """
    json_node = {'name': node.name}
    children = []
    for c in node.getChildCategories():
        children.append(generate_classification(c))
    for s in node.getChildServices():
        children.append({'name':s.name})
    if len(children)>0:
        json_node['children']=children
    return json_node

class CustomResolver(etree.Resolver):
    """
    CustomResolver is a Resolver for lxml that allows (among other things) to
    handle HTTPS protocol, which is not handled natively by lxml/libxml2
    """
    def resolve(self, url, id, context):
        return self.resolve_file(urllib.urlopen(url), context)

parser = etree.XMLParser(no_network=False)
parser.resolvers.add(CustomResolver())

def process(self):
    """
    The overloaded method that performs the processing of results in this web service
    """
    
    # Fetch the form request handler
    form = self.request
    # Fetch the action attribute - all the calls to the service must be associated 
    # with a unique action value -e.g form_submit, get_portal_config, etc..
    action_name = form['action'].value
    
    # The output message content
    self.message = ''
    
    # session id
    sess_id = self.sessionKey
    self.anonymousSession = self.session
    
    if action_name == 'get_portal_config':
        # used by BMPS to get the Mobyle authentication possibilities according to the current configuration
        self.mime_type="application/json"
        self.message = '{"anonymous_session":"%s","authenticated_session":"%s"}' % (self.cfg.anonymousSession(), self.cfg.authenticatedSession())
    if (action_name != 'login' and action_name != 'logout') :
        # performs initialization steps when the actions are not related to login 
        if self.session is not None:
            tmp_data_dir = os.path.realpath( os.path.join(self.session.getDir(), 'BMW' ))
            # Bug: This needs to be changed. Workflow name should be prepended to the
            # file or the better solution is to directly save the content to the
            # workflow XML instead of writing to the tasks.xml - Vivek gopalan 15NOV2011
            wfSessionDirName = AuthenticatedSession.DIRNAME if self.session.isAuthenticated() else AnonymousSession.DIRNAME
            if not os.path.exists(tmp_data_dir):
                os.makedirs( tmp_data_dir, 0755 ) #create parent directory
            tmp_url = "%s/%s/%s/%s" % ( self.cfg.user_sessions_url(),wfSessionDirName,sess_id,'BMW')
    if action_name == 'graphviz_layout' :
        # returns the graphviz layout for the workflow in graphml format
        # action=graphviz_layout&wf_name=xxxx[&graphml_content=xxx&output_graphml=1
        wf_name = form['wf_name'].value
        wff = BMPSWorkflow(wf_name, self.session)
        self.message = wff.graphviz_layout()

    elif action_name =='login':
        # BMPS and BMID action for login to the Mobyle system
        # action=login&email=xx@gmail.com&password=xxxx
        #
        # replies a JSON message: 
        # 'status' is true/false according to authentication success/fail
        # 'message' is an optional error message (if authentication fails)
        # 'source' is the invalid field name (interface should focus on it)
        self.mime_type="application/json"
        response = {'status':True}
        email          = form.getfirst('email')
        password       = form.getfirst('password')
        create_account = False
        try:
            mobyle_email = Mobyle.Net.EmailAddress( email )
        except MobyleError:
            response = {'status':False, 'source' : 'email','message': 'Invalid e-mail, please provide a valid address'}
        else:
            if form.has_key('create_account') and form.getfirst('create_account') == 'yes' :
                create_account = True 
                if (create_account == True) :
                    try:
                        self.session = self.sessionFactory.createAuthenticatedSession( mobyle_email, password)
                    except Exception, e:
                        mb_cgi.c_log.error("error creating a session: %s" % str(e), exc_info=True)
            try:
                self.anonymousSession = self.session
                self.session = self.sessionFactory.getAuthenticatedSession(mobyle_email,passwd=password)
                if self.anonymousSession and self.anonymousSession != self.session:
                    self.session.mergeWith(self.anonymousSession)
                self.sessionKey = self.session.getKey()
                self.read_session()
                self.message = str(True)
            except SessionError, e:
                response = {'status':False, 'source' : 'email','message': str(e)}
        self.message = simplejson.dumps(response, encoding='ascii')

    elif action_name =='validate_guest_access':
        # validate_guest_access = provide an email and/or captcha if necessary to submit
        # jobs as a guest
        # replies a JSON message: 
        # 'status' is true/false according to authentication success/fail
        # 'message' is an optional error message (if authentication fails)
        # 'source' is the invalid field name (interface should focus on it)
        self.mime_type="application/json"
        email          = form.getfirst('email')
        captcha_answer       = form.getfirst('captcha_answer')
        response = {'status':True}
        try:
            self.session.setEmail(Mobyle.Net.EmailAddress(email))
        except MobyleError:
            response = {'status':False, 'source' : 'email','message': 'Invalid e-mail, please provide a valid address'}
        else:
            try:
                ok = self.cfg.anonymousSession()=='yes' or self.session.checkCaptchaSolution(captcha_answer)
            except MobyleError:
                ok = False
            if not(ok):
                response = {'status':False, 'source' : 'captcha','message': 'Wrong answer, try again'}
        self.message = simplejson.dumps(response, encoding='ascii')
    elif action_name =='logout':
        # Used in BMPS and BMID module to logout and delete the current session
        try:
            # Check if non-anonymous session and then assigns the current session to 
            # anonymous session
            if (self.cfg.anonymousSession()!='no'):
                self.session = self.sessionFactory.getAnonymousSession()
                self.sessionKey = self.session.getKey()
            else:
                self.session = None
                self.sessionKey = None
            self.read_session()
            self.message = str(True)
        except SessionError, e:
            self.message  = str(False)
            self.message  += " - " +str(e)
    elif action_name == 'list_workflow_with_description':
        # Used in BMPS to list all the workflow name with the description
        if self.session is not None:
            session_folder = self.session.getDir()
            workflow_list = list_workflow_with_description(session_folder)
            self.message = simplejson.dumps(workflow_list)
    elif action_name == 'send_job_report' :
        # Used in BMPS to send the email about the job report. Job id must be present in the form request 
        # action=send_job_report&id=<job_id>&message="Job did not run"&param="
        job_id = self.request.getfirst('id',None)
        job_url = None
        if id is not None:
            try:
                job_url = registry.getJobURL(job_id)
            except KeyError:
                pass
        emailHelpRequest(self.cfg, self.session.getEmail(), \
                         registry, job_url, \
                         form.getfirst('message',None), self.session, \
                         self.request.getfirst('param',None), \
                         self.request.getfirst('message',None)) 
        self.message = "True"
    elif action_name == 'get_wf' :
    # Used in BMPS to fetch a specific workflow in graphml format. GraphML format is used in the 
    # Client-side to display the workflow in the web interface
    #action=get_wf&wf_name=xxx
        self.message = "<graphml/>";
        if self.session is not None:
            wf_name = form['wf_name'].value
            wff = BMPSWorkflow(wf_name, self.session)
            wf_name = form['wf_name'].value
            self.message = wff.get_graphml()
    elif action_name == 'upload_wf_graphml' :
    # Used in the BMPS to save the workflow to the session based on the graphML content 
    # action=upload_wf_graphml&wf_name=xxxx&graphmla=<graphml content>
        wf_name = form['wf_name'].value
        # Create the BMPSWorkflow object
        wff = BMPSWorkflow(wf_name, self.session)
        # update the graphml content
        wff.update_graphml(form['graphml'].value)
        self.message = "workflow saved successfully: <a href ='" + tmp_url + '/' + wff.mobylexml_filename + "' target='_blank'>" + wff.mobylexml_filename + "</a>"
        
    elif action_name == 'delete_wf' :
        # Used in the BMPS to delete the workflow
        # action=delete_wf&wf_name=xxxx
        wf_name = form['wf_name'].value
        wff = BMPSWorkflow(wf_name, self.session)
        wff.delete()
        self.message = "SUCCESS"
    elif action_name == 'rename_wf' :
        # Used in the BMPS to rename the workflow name
        # action=rename_wf&wf_from_name=xxx&wf_to_name=yyy
        self.mime_type="application/json"
        old_name = self.request.getfirst('wf_from_name')
        new_name = self.request.getfirst('wf_to_name')
        wff = BMPSWorkflow(old_name, self.session)
        status = True
        message = "everything is ok" # default error message
        if old_name!=new_name:
            try:
                wff.rename_wf(new_name, self.request.getfirst('wf_description'))
            except RenameError, r_e:
                status = False
                message = r_e.message
        else:
            wff.change_wf_description(self.request.getfirst('wf_description'))
        self.message = simplejson.dumps({'status':status,'message':message}, encoding='ascii')
    elif action_name == 'get_apps_class_future':
        self.mime_type="application/json"
        p_classification = ClassificationIndex("program")
        #TODO: before to use the workflows:
        # - they should be made "draggable" into the canvas area (currently it crashes the editor)
        # - the issue of user defined workflows appearing there should be solved
        #w_classification = ClassificationIndex("workflow")
        p_classification.buildRegistryCategories(field=self.request.getfirst('classifyBy','category'),serviceTypeSort=self.request.getfirst('serviceTypeSort','separate'))
        #w_classification.buildRegistryCategories(field=self.request.getfirst('classifyBy','category'),serviceTypeSort=self.request.getfirst('serviceTypeSort','separate'))
        registry.name = "All apps"
        programs = [c for c in registry.getChildCategories() if c.name=="Programs"][0]
        classification =generate_classification(programs)
        self.message = simplejson.dumps({"status":"success","details":classification,"message":"success"}, encoding='ascii')
    elif action_name == 'get_apps_class' :
        self.mime_type="application/json"
        s_type = 'program'
        field ='category'
        if form.has_key('format') and form.getfirst('format') == 'app_names_only':
            app_names_list=[p.name for p in registry.programs]
            app_names_list.sort(key=string.lower) # sort alphabetically
            self.message += '{"status":"success","details": {"files": ["' + '","'.join(app_names_list)+ '"]},"message":"success"}'
        else:
            # fetch the Classification index (generated by mobdeploy command)
            p_classification = ClassificationIndex(s_type)
            index = p_classification.index
            classifications = []
            for s in getattr( registry, s_type + 's'):
                cats = index[s.url][field]
                for cn in cats:# for each classification of the program
                    out = s.name.strip()+ ':' + cn.strip()
                    classifications.append(out)
            if form.has_key('app_name'):
                app_name = form.getfirst('app_name')
                param_name = form.getfirst('param_name')
                classification_list = get_pipeout_params(app_name, param_name,classifications)
                root_node1 = get_classification(classification_list)
                self.message += '{"status":"success","details": ' + root_node1.to_json_str() + ',"self.message":"success"}'
            else:
                root_node = get_classification(classifications)
                self.message += '{"status":"success","details": ' + root_node.to_json_str() + ',"message":"success"}'
    elif action_name == 'check_pipeins' :
            # Action check whether the status of various input parameters can be piped to the selected input program's parameter
            # action=check_pipeins&app_name=bowtie&param_name=output&params=tophat:input,soapsnp:input_bam
            app_name = form.getfirst('app_name')
            param_name = form.getfirst('param_name')
            params = form.getfirst('params')
            out = get_pipeout_status(app_name,param_name, params)
            self.message += out

    elif action_name == 'form_submit' :
        # BMPS Action to save a specific program parameters in the Mobyle workflow as BMPSWorkflow object
        # action=form_submit&wf_name=xxx&app_id=bowtie_14555
        task_id = form.getfirst('app_id')
        wf_name = form['wf_name'].value
        wff = BMPSWorkflow(wf_name, self.session)
        wff.set_task_values(task_id, form)
        self.message = "Saved successfully.."
    elif action_name == 'get_datatype' :
        # Action used in BMID to get all the class and superclass datatypes of the Mobyle applications
        # The output is a JSON object with the class and superclass contents
        # action=get_datatype
        data_types = {}
        class_types = set()
        class_superclass_types = set()
        #data_types["class_only"]={}
        data_types["class_superclass"]={}
        si = SearchIndex("program")
        servicesList = getattr( registry, si.type + 's')[:]
        for s in servicesList:
            s.searchMatches = []
            param_class = param_superclass = None
            if si.index.has_key(s.url):
                for field, value in si.index[s.url].items():
                    if field == 'parameter class':
                        param_class = value[0]
                    if field == 'parameter superclass':
                        param_superclass = value[0]
                if not param_superclass :
                    class_types.add(param_class)
                else :
                    class_superclass_types.add(param_class + ":" + param_superclass)
                    #data_types["class_superclass"][param_class] = param_superclass

        #data_types["class_only"] = list(class_types)
        data_types["class_superclass"] = sorted(list(class_types)) + sorted(list(class_superclass_types))
        self.message = data_types
 
    elif action_name == 'get_app_xml':
        # BMPS action to fetch the mobyle XML of a specific application
        # This action is used to fetch Mobyle Program XML require for BMID and BMPS Client side form rendering.
        # The interface and layout tags are removed from main program XML since it is not required by the BMID or Workflow projects
        #  1. When app_id and wf_name are provided then the Program XML files is returned with for the program associated with the app_id
        #      action=get_app_xml&wf_name=test&app_id=3434243&app_name=bowtie
        #  2. When app_id and job_id are specified then the Program XML files is returned for the program name associated with the workflow job id
        #     the vdef tags in parameters filled with values from job results data.
        #      action=get_app_xml&job_id=test.1343&app_id=3434243&app_name=bowtie
        if form.has_key('wf_name'):
            # Process wf_name (used before job submission)
            task_id = form.getfirst('app_id')
            app_name = form.getfirst('app_name')
            wf_name = form['wf_name'].value
            wff = BMPSWorkflow(wf_name, self.session)
            self.message=wff.get_task_xml(task_id,app_name)
            return
        else:
            values = None
            if form.has_key('job_id'):
                # Process workflow job (used after job submission)
                job_id = form.getfirst('job_id')
                values = []
                jobUrl = registry.getJobURL(job_id);
                job = self.session.getJob(jobUrl);
                jobState = JobState(jobUrl)
                params = []
                input_job_files = jobState.getInputFiles()
                # Process all the parameters
                for param_name,value in jobState.getArgs().items() :
                    isResubmit = False
                    # Process the input file separately
                    # Since Multiple* parameters can return multiple values
                    # parameter values are returned as array
                    for file_param_name,file_values in input_job_files :
                        if (file_param_name == param_name) :
                            value =[]
                            for file_name in [ x[0] for x in file_values] :
                                value1 = file_name
                                for data_id in job['dataUsed']:
                                    data = self.session.getData(data_id)
                                    if data['userName'] == value1 :
                                        isResubmit = True
                                        value.append(data_id + ',' + job['jobID'] + '/' + value1 + ',' +  value1)
                    values.append((param_name,value,isResubmit))
            app_name = form.getfirst('app_name')
            app_xml_file = registry.getProgramPath(app_name)
            program_node = set_default_value(app_xml_file,values)
            self.message = etree.tostring(program_node)
    elif action_name == 'get_job_graphml':
        # This method is used in BMPS to get the worflow in GraphML format based on the input job ID.
        # i.e. this action is used to fetch and render workflow diagram in the client after a job is submitted
        # action=get_job_graphml&id=<job id>
        wfId =self.request.getfirst('id',None)
        self.message = get_formatted_job_entries(wfId,self.session)


def set_default_value(app_xml_file,task_inp_values):
    # This method removes the vdef element values in the parameters/values specified in the
    # task_input_values list and then add the values in the task_input_values for the
    # parameters as vdef element value.
    # the modified program Mobyle XML content is sent to the client for rendering by GWT libraries in
    # Mobyle-client libraries
    program_node = None
    if (task_inp_values is not None):
        program_node = etree.parse(app_xml_file)
        # 1. Remove the already existing vdef elemet values in the Mobyle program XML
        for param_name, value, isResubmit in list(task_inp_values):
            params = program_node.xpath(".//parameters/parameter[name=\"%s\"]" % param_name)
            for param in params:
                vdef_node = param.find('vdef')
                if vdef_node is not None:
                    param.remove(vdef_node)
        # 2. Create vdef element and add values based on the input task values
        for param_name, value,isResubmit in list(task_inp_values):
            vdef_new = etree.Element("vdef")
            # Multiple* Parameter values are returned as list
            if isinstance(value, list) :
                tmp_values = value;
            else :
                tmp_values = [value]
            for v in tmp_values :
                value_node = etree.Element("value")
                value_node.text = v
                vdef_new.append(value_node)
            # 3. Add the new vdef element to the program
            params1 = program_node.xpath(".//parameters/parameter[name=\"%s\"]" % param_name)
            for param1 in params1:
                param1.append(vdef_new)
            if isResubmit :
                param1.set('isResubmit','1')
    else:
        if os.path.exists(app_xml_file):
            program_node = etree.parse(app_xml_file)
        if program_node is not None:
            # removed elements not required for the BMID and Workflow client side form rendering
            # This save some network bandwidth
            _remove_from_parent(program_node,".//interface")
            _remove_from_parent(program_node,".//layout")
            _remove_from_parent(program_node,".//parameter/[name='stdout']")
            _remove_from_parent(program_node,".//parameter/[name='stderr']")
    return program_node

def _remove_from_parent(node,xpath_str) :
    # private method to remove XML elements from the parent nodes
    # the XML elements are obtained based on the XSL search on the input XML Node object
    eles = node.findall(xpath_str)
    for el in eles:
        el.getparent().remove(el)

def get_formatted_job_entries(wfId,session):
    try:
        wfUrl = registry.getJobURL(wfId)
        wfJob = session.getJob(wfUrl)
        jf = JobFacade.getFromJobId(wfJob[ 'jobID' ])
        wfState = JobState(wfUrl)
        wfJob['subjobs'] = jf.getSubJobs()
        graphmlAbsPath = os.path.join(url2path(wfJob['jobID']), graphml_file_name)
        doc = etree.parse(graphmlAbsPath, parser)
        root = doc.getroot()
        for data_ele in root.xpath("graphml/graph/data[@key='workflowJobStatus']"):
            data_ele.getParent().remove(data_ele)
        if (wfJob['status']) :
            for data_ele in root.xpath('/graphml/graph'):
                dataNode = etree.Element("data")
                dataNode.set("key", "workflowJobStatus")
                dataNodeStatus = etree.Element("value")
                dataNodeStatus.text = str(wfJob['status'])
                dataNodeMessage = etree.Element("message")
                dataNodeMessage.text = wfJob['status'].message
                dataNode.append(dataNodeStatus)
                dataNode.append(dataNodeMessage)
                data_ele.append(dataNode)
        for data_ele in root.xpath("/graphml/graph/node/data[@key='processJobStatus']"):
            data_ele.getParent().remove(data_ele)
            dataNode = etree.Element("data")
            dataNode.set("key","processJobStatus")
            dataNodeStatus = etree.Element("value")
            dataNodeStatus.text = "undefined"
            dataNode.append(dataNodeStatus)
            data_ele.getParent().append(dataNode)
        if wfJob.has_key('subjobs') and wfJob['subjobs'] is not None:
            for subjob in wfJob['subjobs']:
                jobUrl = subjob['jobID']
                taskRef = wfState.root.xpath( 'jobLink[@jobId="%s"]/@taskRef' % jobUrl)
                if ( taskRef and len(taskRef) > 0) :
                    for data_ele in root.xpath('/graphml/graph/node[@id="%s"]' % taskRef[0]):
                        dataNode = etree.Element("data")
                        dataNode.set("key","processJobStatus")
                        dataNodeStatus = etree.Element("value")
                        dataNodeStatus.text = str(subjob['status'])
                        dataNodeMessage = etree.Element("message")
                        dataNodeMessage.text = subjob['status'].message
                        dataNode.append(dataNodeStatus)
                        dataNode.append(dataNodeMessage)
                        data_ele.append(dataNode)
                    for data_ele in root.xpath('/graphml/graph/node[@id="%s"]' % taskRef[0]):
                        dataNode = etree.Element("data")
                        dataNode.set("key","jobID")
                        dataNode.text = str(subjob['jobPID'])
                        data_ele.append(dataNode)
        for data_ele in root.xpath("/graphml/graph/node[not(data/@key='processJobStatus')]"):
            dataNode = etree.Element("data")
            dataNode.set("key","processJobStatus")
            dataNodeStatus = etree.Element("value")
            dataNodeStatus.text = "undefined"
            dataNode.append(dataNodeStatus)
            data_ele.append(dataNode)
        return etree.tostring(root,method="xml", pretty_print=True)
    except Exception:
        return "<graphml/>"

def sorted_ls(path):
    """
    Utility method to sort the file contents in a specific directory by time
    """
    mtime = lambda f: os.path.getctime(os.path.join(path, f))
    return list(sorted(os.listdir(path), key=mtime))

def get_workflow_description(session_folder, workflow_name):
    """
    Utility method to the description associated with the workflow short-name from session folder
    The description is obtained from the graphml's description element tag.

    workflow_name: workflow name
    return: description
    """
    source_folder = os.path.realpath(os.path.join(session_folder, 'BMW'))
    source_filename = os.path.realpath(os.path.join(source_folder, workflow_name + '.graphml'))
    if not os.path.exists(source_filename):
        return ''
    source_file = open(source_filename, 'r')
    description = ''
    try:
        root_tree = etree.parse(source_file)
        element = root_tree.find('graph')
        description = element.get('description')
        if description is None:
            description = ''
    except Exception:
        pass
    finally:
        source_file.close()
    return description

def list_workflow_with_description(session_folder):
    """
    Utility method to fetch the workflow names with its description from the session folder
    (A better BMPSWorkflow based solution should be provided in future - Vivek Gopalan 09FEB2013)
    
    session_folder: user session folder
    return: list of workflow name and list of workflow description
    """
    source_folder = os.path.realpath(os.path.join(session_folder, 'BMW'))
    sorted_filename_list = sorted_ls(source_folder)
    pattern = re.compile('\.graphml$', re.IGNORECASE)
    workflow_name_list = []
    workflow_description_list = []
    # Get all the files in the session folder tha ends with graphml
    for filename in sorted_filename_list :
        if pattern.search(filename):
            workflow_name = re.sub('\.graphml$', '', filename)
            workflow_name_list.append(workflow_name)
            # fetch description from graphml file
            workflow_description = get_workflow_description(session_folder, workflow_name)
            workflow_description_list.append(workflow_description)
    return [workflow_name_list, workflow_description_list]

if __name__ == "__main__":
    # the main entry class for this server
    mb_cgi.HTMLCGI(processFunction=process,useSession=True)
