View Javadoc

1   /*
2    * $Id: CommandLineApplication.java,v 1.18 2004/12/18 15:43:57 jdt Exp $
3    *
4    * Created on 14 October 2003 by Paul Harrison
5    * Copyright 2003 AstroGrid. All rights reserved.
6    *
7    * This software is published under the terms of the AstroGrid
8    * Software License version 1.2, a copy of which has been included
9    * with this distribution in the LICENSE.txt file.
10   */
11  
12  package org.astrogrid.applications.commandline;
13  
14  import org.astrogrid.applications.AbstractApplication;
15  import org.astrogrid.applications.ApplicationExecutionException;
16  import org.astrogrid.applications.CeaException;
17  import org.astrogrid.applications.DefaultIDs;
18  import org.astrogrid.applications.Status;
19  import org.astrogrid.applications.beans.v1.parameters.ParameterValue;
20  import org.astrogrid.applications.description.ApplicationInterface;
21  import org.astrogrid.applications.description.ParameterDescription;
22  import org.astrogrid.applications.parameter.ParameterAdapter;
23  import org.astrogrid.applications.parameter.protocol.ExternalValue;
24  import org.astrogrid.applications.parameter.protocol.ProtocolLibrary;
25  import org.astrogrid.workflow.beans.v1.Tool;
26  
27  import org.apache.commons.collections.IteratorUtils;
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  
31  import java.io.BufferedReader;
32  import java.io.File;
33  import java.io.FileNotFoundException;
34  import java.io.FileOutputStream;
35  import java.io.FileReader;
36  import java.io.FilterReader;
37  import java.io.IOException;
38  import java.io.Reader;
39  import java.util.AbstractList;
40  import java.util.ArrayList;
41  import java.util.Collections;
42  import java.util.Comparator;
43  import java.util.Iterator;
44  import java.util.List;
45  /***
46   * A generic model for a command line application. This generally assumes that the application can be run from a command line obtaining all of its parameters from commandline arguments and possibly standard in. 
47   * The application can interact with the filesystem.
48   * 
49   * This is achieved with the {@link java.lang.Runtime#exec(java.lang.String[], java.lang.String[], java.io.File) }call,
50   * 
51   * It was programmed/tested under unix, but I think that the code would work with windows based systems with very little alteration.
52   * @author Paul Harrison (pah@jb.man.ac.uk)
53   * @version $Name:  $
54   * @since iteration4
55   */
56  public class CommandLineApplication extends AbstractApplication implements Runnable {
57      /*** time to wait for the outstreams to finish writing in milliseconds
58      */
59     private static final int OUTSTREAM_WAITTIME = 2000;
60     /***
61       * Commons Logger for this class
62       */
63      private static final Log logger = LogFactory.getLog(CommandLineApplication.class);
64  
65     private Thread exeThread;
66     private StreamPiper outPiper;
67     private StreamPiper errPiper;
68     
69     private String[] args = null;
70     protected final List argvals = new ArrayList();
71     protected String[] envp = null;
72     protected final Runtime runtime = Runtime.getRuntime();
73     protected Process process;
74     protected int exitStatus;
75  
76     protected final CommandLineApplicationEnvironment applicationEnvironment;
77  
78     /***
79      * @param controller
80      * @param user
81      */
82     public CommandLineApplication(String jobStepId,  Tool t,ApplicationInterface interf, CommandLineApplicationEnvironment env,ProtocolLibrary lib)  {
83        super(new DefaultIDs(jobStepId,env.getExecutionId()),t, interf,lib);
84        this.applicationEnvironment = env;
85     }
86  
87     public boolean execute() throws CeaException {
88          logger.info("creating new thread to execute.. " + this.toString());
89          
90          exeThread = new Thread(CommandLineApplication.this);
91          exeThread.start();
92          return true;
93      }
94  
95     
96     
97      public Runnable createExecutionTask() throws CeaException {
98        return this;
99      }
100     /*** override  so that commandline parameters are returned
101      * @see org.astrogrid.applications.AbstractApplication#instantiateAdapter(org.astrogrid.applications.beans.v1.parameters.ParameterValue, org.astrogrid.applications.description.ParameterDescription, org.astrogrid.applications.parameter.indirect.IndirectParameterValue)
102      */
103     protected ParameterAdapter instantiateAdapter( ParameterValue pval, ParameterDescription desr, ExternalValue indirectVal) {                
104         CommandLineParameterDescription clpd = (CommandLineParameterDescription)desr;
105              logger.debug("creating parameter adapter for " + pval.getName());
106              return new DefaultCommandLineParameterAdapter(getApplicationInterface(),pval, (CommandLineParameterDescription)desr,indirectVal,applicationEnvironment);
107          
108       }
109    protected void setupParameters() throws CeaException {
110       // just setup the actual command line for now
111       reportMessage("Setting up parameters");
112       argvals.clear();
113       argvals.add(((CommandLineApplicationDescription)getApplicationDescription()).getExecutionPath());
114 
115     createAdapters();
116       // allow last minute manipulation of parameters before the application runs
117       reportMessage("Calling postParamSetupHook");
118      postParamSetupHook();
119      reportMessage("postParamSetupHook - completed");
120      
121      //create a list of all the parameter adapters and sort them.
122      List allAdapterList = IteratorUtils.toList(parameterAdapters());
123      Collections.sort(allAdapterList, new Comparator(){
124        public int compare(Object o1, Object o2) {
125            //FIXME - need to check that the ordering of "equal" objects stays the same with thiss algorithm
126             DefaultCommandLineParameterAdapter p1 = (DefaultCommandLineParameterAdapter)o1;
127             DefaultCommandLineParameterAdapter p2 = (DefaultCommandLineParameterAdapter)o2;
128             int pos1 = p1.cmdParamDesc.getCommandPosition();
129             int pos2 = p2.cmdParamDesc.getCommandPosition();
130             if(pos1 == -1) // indicates that it is not a position dependent
131                           // parameter
132             {
133                 if(pos2 == -1)
134                 {
135                     return 0; // they are equivalent
136                 }
137                 else
138                 {
139                     return 1; // p2 should be before p1
140                     }
141             }
142             else // it is a position dependent parameter
143             {
144                 if (pos2 == -1) {
145                     return -1;  // the positional parameters should come first
146                 }
147                 else {
148                     if (pos1 == pos2)
149                     {
150                         return 0; 
151                     }
152                     else
153                     {
154                         return pos1 > pos2 ? 1 : -1;
155                     }
156                     }
157                 }
158                     
159             };
160         }
161      );
162      
163      // iterate over all the parameter adapters now that they have been sorted...
164      for (Iterator i = allAdapterList.iterator(); i.hasNext(); ) {
165          ParameterAdapter adapter = (ParameterAdapter)i.next();
166              adapter.process();
167       }
168      
169      //iterate over the paramters adapters adding the commandline switches
170      for (Iterator i = allAdapterList.iterator(); i.hasNext();) {
171         CommandLineParameterAdapter adapter = (CommandLineParameterAdapter)i.next();
172         List vals = adapter.addSwitches();
173         if (vals != null) {
174             for (Iterator j = vals.iterator(); j.hasNext();) {
175                 argvals.add(j.next().toString());
176             } 
177         }                  
178     }
179      
180      
181      reportMessage("Parameters for application: " + argvals.toString());
182    }
183 
184    private void startApplication() throws ApplicationExecutionException {
185         FileOutputStream stdout, stderr;
186         logger.info("Starting Application " +this.toString());
187         try {
188            stdout = new FileOutputStream(applicationEnvironment.getOutputLog());
189            stderr = new FileOutputStream(applicationEnvironment.getErrorLog());
190         }
191         catch (FileNotFoundException e) {
192           reportError("Could not find required logs", e);
193            throw new ApplicationExecutionException("Cannot open stdout or stderr files", e);
194         }
195         try {
196            args = (String[])argvals.toArray(new String[0]);
197            logger.debug("Args will be " + argvals.toString());
198            setStatus(Status.RUNNING);
199            process =  runtime.exec(args, envp, applicationEnvironment.getExecutionDirectory());
200         }
201         catch (IOException e1) {
202            reportError("IOException when running app.", e1);
203            throw new ApplicationExecutionException("Cannot create the application process", e1);
204         }
205       
206         errPiper = new StreamPiper("err", process.getErrorStream(), stderr);
207 
208         //this call to process.getInputStream has to be a bug in the JDK 0
209         outPiper = new StreamPiper("out", process.getInputStream(), stdout);
210      }
211 
212 
213     /***
214      * run a thread runs and waits for the application.
215      */
216     public void run() {
217         try {
218             try {
219                 setupParameters();
220             }
221             catch (CeaException e) {
222                 reportError("problem setting up parameters", e);
223                 return;
224             }
225             reportMessage("Calling preRunHook");
226             preRunHook();
227             reportMessage("PreRunHook - completed");
228             try {
229                 startApplication();
230 
231             }
232             catch (ApplicationExecutionException e1) {
233                 reportError("problem running application", e1);
234                 return;
235             }
236 
237             reportMessage("waiting for " + this.toString() + " to finish....");
238             try {
239                 exitStatus = process.waitFor();
240                 reportMessage(this.toString() + " finished");
241                 endApplication();
242             }
243             catch (InterruptedException e) {
244                 // TODO need to work out if it was interrupted on purpose and do
245                 // something appropriate...
246                 reportError("application " + this.toString()
247                         + " was interrupted", e);
248             }
249         }
250         catch (Throwable e) {
251             reportError("unexpected runtime error in application"
252                     + this.toString() +" "+ ids.getId()+" "+ids.getJobStepId() , e);
253         }
254     }
255 
256 /***
257   *stop reader and writer threads and free up some resources 
258   
259  * @throws CeaException
260  */
261 private final void endApplication()  {
262       reportMessage("Execution post application processes");
263       //wait for the error stream a little....
264       errPiper.join(OUTSTREAM_WAITTIME);
265       errPiper.terminate();
266       outPiper.terminate();
267       process = null;
268       
269       setStatus(Status.WRITINGBACK);
270       // call the hook to allow manipulation by subclasses
271       reportMessage("Calling preWritebackHook");
272       preWritebackHook();
273       reportMessage("preWritebackHook - completed");
274       // copy back any output parameters
275       ApplicationInterface applicationInterface = getApplicationInterface();
276       for (Iterator i = outputParameterAdapters(); i.hasNext(); ) {   
277          ParameterAdapter adapter = (ParameterAdapter)i.next();
278          try {         
279             adapter.writeBack(null);
280          } catch (CeaException e) {                        
281                 reportWarning("There was a problem writing back parameter "+adapter.getWrappedParameter().getName(),e);
282                 //set non-zero exit status if not already set to force the reporting of standard error below....
283                 exitStatus = exitStatus == 0? -1 : exitStatus;
284          }
285       }        
286 
287       reportMessage("The application has completed with exit status="+exitStatus);
288       if (exitStatus != 0) {
289          reportStandardError(true);// send the stderr output as well
290 //         setStatus(Status.ERROR); the reporting will automatically do this....TODO need to refactor how the writing to permanent store is signaled....
291       } else {
292           setStatus(Status.COMPLETED);//it notifies that results are ready to be consumed.
293       }
294    }
295    
296    /***
297  * Report the standard error output to the listeners.
298  */
299 private void reportStandardError(boolean doError) {
300  
301    File logfile;
302    String streamName;
303    if(doError)
304    {
305       logfile = applicationEnvironment.getErrorLog();
306       streamName="error";
307    }
308    else
309    {
310       logfile = applicationEnvironment.getOutputLog();
311       streamName="output";
312    }
313 
314    try {
315        
316       BufferedReader errReader = new BufferedReader( new FileReader(logfile));
317         StringBuffer errMsg = new StringBuffer();
318         String line;
319         while((line = errReader.readLine()) != null)
320         {
321            //FIXME - need to filter the strings
322            line = line.replace('\u001b', ' '); //get rid of escape characters...
323            errMsg.append(line);
324            errMsg.append('\n');
325         }
326         //TODO - need to think about limiting the size of the returned error messages...
327         reportError("The standard "+streamName+" from the command line application follows\n"+errMsg.toString());
328         errReader.close();
329     }
330     catch (IOException e) {
331        reportError("cannot write back standard "+streamName, e);
332     }
333 }
334 
335 /***
336      * Hook to allow for manipulation of the environment before the application gets run. Occurs when the command line has been built for the application. This does nothing in the default implementation.
337      *
338      */
339     protected void preRunHook() {
340        // do nothing special by default
341     }
342 
343 
344    /***
345     *  Hook to allow special parameter processing. This occurs after the parameters have had their default treatment. This does nothing in the default implementation.
346     */
347    protected void postParamSetupHook() {
348       // do nothing
349    }
350 
351 
352 
353    /***
354     * Hook to allow the manipulation of results before they are written back to the caller. This does nothing in the default implementation.
355     */
356    protected void preWritebackHook() {
357       // default is to do nothing special
358    }
359 
360    /***
361     * @return
362     */
363    public CommandLineApplicationEnvironment getApplicationEnvironment() {
364       return applicationEnvironment;
365    }
366 
367 
368 
369 
370    /* (non-Javadoc)
371     * @see org.astrogrid.applications.Application#completionStatus()
372     */
373     /* or this
374    public int completionStatus() {
375       return exitStatus;
376    }
377    */
378 
379 
380 
381 
382    
383 }