View Javadoc

1   /*$Id: JesShell.java,v 1.17 2004/12/03 14:47:41 jdt Exp $
2    * Created on 29-Jul-2004
3    *
4    * Copyright (C) AstroGrid. All rights reserved.
5    *
6    * This software is published under the terms of the AstroGrid 
7    * Software License version 1.2, a copy of which has been included 
8    * with this distribution in the LICENSE.txt file.  
9    *
10  **/
11  package org.astrogrid.jes.jobscheduler.impl.groovy;
12  
13  import org.astrogrid.applications.beans.v1.parameters.ParameterValue;
14  import org.astrogrid.community.User;
15  import org.astrogrid.community.beans.v1.Credentials;
16  import org.astrogrid.scripting.Toolbox;
17  import org.astrogrid.store.Ivorn;
18  import org.astrogrid.workflow.beans.v1.For;
19  import org.astrogrid.workflow.beans.v1.If;
20  import org.astrogrid.workflow.beans.v1.Input;
21  import org.astrogrid.workflow.beans.v1.Output;
22  import org.astrogrid.workflow.beans.v1.Parfor;
23  import org.astrogrid.workflow.beans.v1.Set;
24  import org.astrogrid.workflow.beans.v1.Tool;
25  import org.astrogrid.workflow.beans.v1.While;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.codehaus.groovy.GroovyException;
30  import org.codehaus.groovy.control.CompilationFailedException;
31  import org.codehaus.groovy.control.CompilerConfiguration;
32  import org.codehaus.groovy.control.messages.WarningMessage;
33  
34  import EDU.oswego.cs.dl.util.concurrent.Callable;
35  import EDU.oswego.cs.dl.util.concurrent.TimedCallable;
36  import groovy.lang.Binding;
37  import groovy.lang.GroovyShell;
38  import groovy.lang.Script;
39  
40  import java.io.IOException;
41  import java.io.PrintStream;
42  import java.lang.ref.SoftReference;
43  import java.net.URISyntaxException;
44  import java.util.Map;
45  
46  import javax.imageio.IIOException;
47  
48  /*** class that encapuslates the execution and evaluation of grouvy code.
49   * <p>
50   * Uses a <a href="http://gee.cs.oswego.edu/dl/classes/EDU/oswego/cs/dl/util/concurrent/TimedCallable.html">Timed Callable</a>
51   * to abort execution after a certain time.
52   * <p>
53   * Added in codebases - to differentiate between user and system script.
54   * @todo find out how to add in security managers to this too.
55   * @author Noel Winstanley nw@jb.man.ac.uk 29-Jul-2004
56   *
57   */
58  public class JesShell {
59      /***
60       * Commons Logger for this class
61       */
62      private static final Log logger = LogFactory.getLog(JesShell.class);
63  
64  
65      /*** Construct a new JesShell
66       * 
67       */
68      public JesShell() {
69          super();
70      }
71      
72      private static final long EVAL_LIMIT = 60 * 1000; // 1 minute
73      private static final long EXEC_LIMIT = 10 * 60 * 1000; // 10 minutes
74  
75      /*** for efficiencies sake, only ever create a single, static interpreter.
76       * this will be ok - as we parse-run all code we pass through it.
77       * plus, we know it is being called in a single thread (the scheduler) only.
78       * however, don't know for sure that this isn't memory leaking. so will be more cautious, and create a new one for each new shell
79       * (i.e. for each workflow processed). but this seems really slow. will implement a kind of pool instead - reuse each object 500 times, then get shot of it.
80       * @modified - made this an soft reference too - so can get reclaimed if things get tight. Done the same with the cached scripts - incase they hold references back to the shell that created them
81       */
82      private static SoftReference shell = new SoftReference(new GroovyShell());
83      private static int useCount = 0;
84      private static final int USE_LIMIT = 500; // as shell gets accessed a lot.
85      private static final CompilerConfiguration config = new CompilerConfiguration();
86      static {
87          config.setVerbose(true);
88          config.setWarningLevel(WarningMessage.PARANOIA);
89      }
90      protected static GroovyShell getGroovyShell() {
91          // logic a little tricky here.
92          GroovyShell s = (GroovyShell)shell.get();
93          if (useCount++ > USE_LIMIT || s == null) {
94              logger.info("Recreating groovy shell");
95              useCount = 0;
96              shell.clear();            
97              s = new GroovyShell(config);           
98              shell = new SoftReference(s);
99          }
100         return s;
101     }
102     
103     //same pattern for the scripting toolbox - as don't know yet how expensive it is to create.
104     private static SoftReference toolbox = new SoftReference(new Toolbox());
105     protected static Toolbox getToolbox() {
106         Toolbox t = (Toolbox)toolbox.get();
107         if (t == null) {
108             logger.info("Recreating toolbox");
109             t = new Toolbox();
110             toolbox = new SoftReference(t);
111         }
112         return t;
113     }
114     
115     protected JesInterface jes;
116     
117 
118     
119     public String evaluateIndex(Script indexScript,ActivityStatusStore statusStore) {
120         Binding triggerBinding = createTriggerBinding(statusStore);
121         indexScript.setBinding(triggerBinding);
122         return (String)indexScript.run();
123     }
124     /*** 
125      * compile a rule script into the rule codebase.
126      * @modified - specifies a code source while compiling.
127      * 
128      * commented out for now - needs further tuning and investigation
129      * */
130     public Script compileRuleScript (String script) throws CompilationFailedException, IOException {
131        /*
132         GroovyCodeSource cs = new GroovyCodeSource( script,"RuleScript","/jes/rule");
133         return getGroovyShell().parse(cs);
134         */
135         return getGroovyShell().parse(script);
136     }
137     
138     /***compile a user script into the user codebase. 
139      * @modified - specifies a code source while compiling.
140      * commented out for now - needs further tuning and investigation
141      * */
142     public Script compileUserScript (String script) throws CompilationFailedException, IOException {
143         /*
144         GroovyCodeSource cs = new GroovyCodeSource( script,"UserScript","/jes/user");
145         return getGroovyShell().parse(cs);
146         */
147         return getGroovyShell().parse(script);
148     }
149     
150     /*
151      * for this to be effective, it seems like I need to instal my own policy- which means
152      * Policy.getPolicy(), wrapping it in a custom subclass, and then Policy.setPolicy()
153      * 
154      * 'course, this relies on the tomcat security policy allowing me to change the policy. gahh.
155      * of course, where the policy is finely configured already, just let the admin edit the policy to give the 
156      * jes script the correct permissions. Need to have this policy-zapper as a separate component, which is allowed to 
157      * fail as needed.
158      */
159     
160     /*** execute the body (concequence) of a rule */
161     public void executeBody(Rule r,ActivityStatusStore statusStore, Map rules) throws CompilationFailedException, IOException {
162         logger.info("firing " + r.getName());
163         Binding bodyBinding = createBodyBinding(statusStore,rules);
164         logger.debug(r);    
165         Script sc = r.getCompiledBody();        
166         if (sc == null) { // not there, compile it up..
167             sc = compileRuleScript( r.getBody());
168             r.setCompiledBody(sc);
169         }
170         sc.setBinding(bodyBinding);
171         sc.run();
172     }
173     /*** execute a script activity 
174      * @throws InterruptedException
175      * @throws IOException
176      * @throws InterruptedException
177      * @throws GroovyException*/
178     public void executeScript(String script,String id, ActivityStatusStore statusStore,Map rules, PrintStream errStream, PrintStream outStream) throws GroovyException, IOException, InterruptedException {
179         logger.info("Running script for id " + id);
180         logger.debug(script);
181         Binding scriptBinding = createScriptBinding(statusStore,rules);
182         Vars vars= statusStore.getEnv(id);
183         vars.addToBinding(scriptBinding);
184         Script sc = compileUserScript(script);
185         sc.setBinding(scriptBinding);
186         PrintStream originalErr = System.err;
187         PrintStream originalOut = System.out;
188         try {
189             System.setErr(errStream);
190             System.setOut(outStream);
191             runWithTimeLimit(sc,EXEC_LIMIT);
192         } finally {
193             System.setErr(originalErr);
194             System.setOut(originalOut);
195         }
196         logger.debug("Completed Script execution");
197         vars.readFromBinding(scriptBinding);
198     }
199     
200     /*** used to evaluate user-supplied expressions - if tests, etc 
201      * @param expr
202      * @param id
203      * @param statusStore
204      * @param rules
205      * @return
206      * @throws IOException
207      * @throws InterruptedException
208      * @throws GroovyException*/
209 public Object evaluateUserExpr(String expr,String id,ActivityStatusStore statusStore, Map rules) throws GroovyException, IOException, InterruptedException {
210     logger.debug("exaluating " + expr);
211     Binding scriptBinding = createScriptBinding(statusStore,rules);
212     Vars vars = statusStore.getEnv(id);
213     vars.addToBinding(scriptBinding);
214     // wrap it in a here-document
215     // want to return a string if it has more thatn just a single embedded ${..}, or is just a simple string.
216     // otherwise want to return the object.
217     StringBuffer gExpr = (new StringBuffer("x = <<<GSTRING\n")) 
218             .append(expr)
219             .append("\nGSTRING\n ")
220             .append("(x instanceof GString && x.getValueCount() == 1 && x.getStrings().find{it.size() > 0} == null) ? x.getValue(0) : x"); 
221    
222     Script sc = compileUserScript(gExpr.toString());
223     sc.setBinding(scriptBinding);    
224     Object result =runWithTimeLimit(sc,EVAL_LIMIT);
225 
226      if (logger.isDebugEnabled()) {
227          logger.debug("result: '" + result + "' type: " + result.getClass().getName());
228      }
229      return result;
230     
231 }
232 /*** wrap a script in a timed callable - will throws an interrupted if script overruns the time 
233  * 
234  * @param sc script to execute
235  * @param timeLimit limit in milliseconds to execute the script for,
236  * @return the result
237  * @throws GroovyException if therer's a problem compiling or executing the script
238  * @throws IOException
239  * @throws InterruptedException if the script execution times out.
240  */
241 protected Object runWithTimeLimit(final Script sc,long timeLimit) throws GroovyException, IOException, InterruptedException {
242     Callable c = new Callable() {
243 
244         public Object call() throws Exception {
245             return sc.run();
246         }
247     };
248     TimedCallable tc = new TimedCallable(c,timeLimit);
249     try {
250         return tc.call();
251     } catch (InterruptedException e) {// propagate upwards
252         throw e;
253     } catch (GroovyException e) {// propagate
254         throw e;
255     } catch (IIOException e) { // propoagate
256         throw e;
257     } catch (Exception e) {// doubt this will occur, but need to handle anyhow.
258         GroovyException x =new GroovyException(e.getMessage());
259         x.initCause(e);
260         throw x;
261     }
262 }
263 
264     /*** executes a 'set' activity 
265      * @throws IOException
266      * @throws InterruptedException
267      * @throws GroovyException*/
268     public void executeSet(Set set,String state,ActivityStatusStore statusStore, Map rules) throws GroovyException, IOException, InterruptedException {
269         Vars vars = statusStore.getEnv(state);
270         if (set.getValue() != null) {
271             Object result = evaluateUserExpr(set.getValue(),state,statusStore,rules);
272             vars.set(set.getVar(),result);
273         } else {
274             // just a declaration, with no initialization.
275             vars.set(set.getVar(),null);
276         }
277     }
278     
279     // necessary to have these, rather than pass the string directly into evaluateUserExpr - 
280    // otherwise the gstirng gets substituted into before it reaches us.
281     /*** evaluates the test of  an 'if' activity 
282      * @throws IOException
283      * @throws InterruptedException
284      * @throws GroovyException*/
285     public Object evaluateIfCondition(If ifObj,ActivityStatusStore statusStore,Map rules) throws GroovyException, IOException, InterruptedException {
286         return evaluateUserExpr(ifObj.getTest(),ifObj.getId(),statusStore,rules);
287         
288     }
289     /*** evaluates the test of a while activity 
290      * @throws IOException
291      * @throws InterruptedException
292      * @throws GroovyException*/
293     public Object evaluateWhileCondition(While whileObj,ActivityStatusStore statusStore,Map rules) throws GroovyException, IOException, InterruptedException {
294         return evaluateUserExpr(whileObj.getTest(), whileObj.getId(),statusStore,rules);
295     }
296     /*** evaluates the items of a for activity 
297      * @throws IOException
298      * @throws InterruptedException
299      * @throws GroovyException*/
300     public Object evaluateForItems(For forObj,ActivityStatusStore statusStore,Map rules) throws GroovyException, IOException, InterruptedException {
301         return  evaluateUserExpr(forObj.getItems(),forObj.getId(),statusStore,rules);
302    
303     }
304     /*** evaluate the items of a parfor activity 
305      * @throws IOException
306      * @throws InterruptedException
307      * @throws GroovyException*/
308     public Object evaluateParforItems(Parfor forObj,ActivityStatusStore statusStore,Map rules) throws GroovyException, IOException, InterruptedException {
309         return  evaluateUserExpr(forObj.getItems(),forObj.getId(),statusStore,rules);
310    
311     }    
312     /*** evaluate the parameter values of a tool */
313     public Tool evaluateTool(Tool original,String id,ActivityStatusStore statusStore, Map rules) throws CompilationFailedException, IOException {
314         Tool copy = new Tool();
315         copy.setInterface(original.getInterface());
316         copy.setName(original.getName());
317         Input input = new Input();
318         copy.setInput(input);
319         
320         Vars vars = statusStore.getEnv(id);
321         Binding scriptBinding = createScriptBinding(statusStore,rules);
322         vars.addToBinding(scriptBinding);
323         if (original.getInput() != null) { // possible we have no inputs, I suppose
324         for (int i = 0; i < original.getInput().getParameterCount(); i++) {
325             ParameterValue originalP = original.getInput().getParameter(i);
326             ParameterValue copyP = processParameter(originalP, scriptBinding);
327             input.addParameter(copyP);
328         }
329         }
330         
331         if (original.getOutput() != null) {
332         // do identical for ouputs.
333         Output output = new Output();
334         copy.setOutput(output);
335         for (int i =0; i < original.getOutput().getParameterCount(); i++) {
336             ParameterValue originalP = original.getOutput().getParameter(i);
337             ParameterValue copyP = processParameter(originalP,scriptBinding);
338             output.addParameter(copyP);
339         }
340         }
341         return copy;
342     }
343 
344 /*** 
345      * @param original
346      * @param scriptBinding
347      * @param i
348      * @return
349      * @throws CompilationFailedException
350      * @throws IOException
351      */
352     private ParameterValue processParameter(ParameterValue originalP, Binding scriptBinding) throws CompilationFailedException, IOException {
353         ParameterValue copyP = new ParameterValue();
354         copyP.setName(originalP.getName());
355         copyP.setIndirect(originalP.getIndirect());
356         copyP.setEncoding(originalP.getEncoding());
357         // evaluate value.. -- always to string.
358         StringBuffer expr = (new StringBuffer("<<<GSTRING\n")).append(originalP.getValue()).append("\nGSTRING\n");
359         Script sc = compileUserScript(expr.toString());
360         sc.setBinding(scriptBinding);
361         Object result = sc.run();
362         copyP.setValue( result.toString());
363         return copyP;
364     }
365 
366     //  binding creation functions - so that scripts can access the status store.
367     private Binding createTriggerBinding(ActivityStatusStore statusStore) {
368         Binding b = new Binding();
369         b.setVariable("states",statusStore);
370         for (int i = 0;  i < Status.allStatus.size() ; i++ ) {
371             Status stat = (Status)Status.allStatus.get(i);
372             b.setVariable(stat.getName(),stat);
373         }
374         return b;
375     }
376 
377     private  Binding createBodyBinding(ActivityStatusStore statusStore,Map rules){
378         Binding b = createTriggerBinding(statusStore);
379         b.setVariable("rules",rules);
380         b.setVariable("jes",jes);
381         b.setVariable("shell",this);
382         return b;
383     }
384     
385     /*** create environment for user scripts to run in - don't provide access to system objects.
386      * @param state
387      * @param statusStore
388      * @return
389      */
390     private Binding createScriptBinding(ActivityStatusStore statusStore, Map rules) {
391         Binding b = new Binding();
392         b.setVariable("jes",jes);
393         for (int i = 0;  i < Status.allStatus.size() ; i++ ) {
394             Status stat = (Status)Status.allStatus.get(i);
395             b.setVariable(stat.getName(),stat);
396         }
397         // basic stuff.
398         JesShell.createBasicScriptBinding(b, getToolbox(), jes.getWorkflow().getCredentials());
399         // ah, what the hell, who knows - it might be useful.
400         b.setVariable("__states",statusStore);
401         b.setVariable("__rules",rules);
402         b.setVariable("__shell",this);
403         
404         
405         return b;
406     }    
407     
408     /***
409      * @param jesInterface
410      */
411     public void setJesInterface(JesInterface jesInterface) {
412         this.jes = jesInterface;
413     }
414     /***
415      * @param b
416      * @param toolbox2 @todo
417      * @param credentials @todo
418      */
419     public static void createBasicScriptBinding(Binding b, Toolbox toolbox2, Credentials credentials) {
420         b.setVariable("astrogrid",toolbox2);
421     
422         b.setVariable("account",credentials.getAccount());
423         String name = credentials.getAccount().getName();
424         String community = credentials.getAccount().getCommunity();
425         User u = new User(name,community,credentials.getGroup().getName(),credentials.getSecurityToken());
426         b.setVariable("user",u);        
427     
428         try {
429             b.setVariable("userIvorn",new Ivorn("ivo://" + community + "/" +  name));
430         } catch (URISyntaxException e) {
431             logger.error("URISyntaxException when creating userIvorn.",e);
432         }
433         b.setVariable("homeIvorn",new Ivorn(community,name,name + "/"));
434         
435     }
436 
437 
438     
439     
440 }
441 
442 
443 /* 
444 $Log: JesShell.java,v $
445 Revision 1.17  2004/12/03 14:47:41  jdt
446 Merges from workflow-nww-776
447 
448 Revision 1.16.2.1  2004/12/01 21:50:53  nw
449 tried to factor out the different parts of the JEScript environment
450 
451 Revision 1.16  2004/11/29 20:00:24  clq2
452 jes-nww-714
453 
454 Revision 1.15.12.5  2004/11/26 13:13:28  nw
455 fix
456 
457 Revision 1.15.12.4  2004/11/26 12:51:30  nw
458 added more useful info into script namespace.
459 
460 Revision 1.15.12.3  2004/11/26 01:31:18  nw
461 updated dependency on groovy to 1.0-beta7.
462 updated code and tests to fit.
463 
464 Revision 1.15.12.2  2004/11/24 18:49:02  nw
465 sandboxing of script execution - first by a timeout,
466 later by java permissions system.
467 
468 Revision 1.15.12.1  2004/11/24 00:23:20  nw
469 get script running in a separate thread, with a timeout (bz#665)
470 worked new scripting objects into environment (bz#715)
471 
472 Revision 1.15  2004/11/05 16:52:42  jdt
473 Merges from branch nww-itn07-scratchspace
474 
475 Revision 1.14.18.1  2004/11/05 16:06:57  nw
476 optimized: cached GroovyShell in softreference
477 optimized: replaced + with stringBuffers
478 removed unused execute-trigger methods
479 added methods to compile / evaluate index code
480 
481 Revision 1.14  2004/09/16 21:43:47  nw
482 made 3rd-party objects only persist for so many calls. - in case they're space leaking.
483 
484 Revision 1.13  2004/09/06 16:47:04  nw
485 javadoc
486 
487 Revision 1.12  2004/09/06 16:30:25  nw
488 javadoc
489 
490 Revision 1.11  2004/08/18 21:50:15  nw
491 improved error propagation and reporting.
492 messages are now logged to workflow document
493 
494 Revision 1.10  2004/08/13 09:10:30  nw
495 tidied imports
496 
497 Revision 1.9  2004/08/09 17:34:10  nw
498 implemented parfor.
499 removed references to rulestore
500 
501 Revision 1.8  2004/08/05 14:38:15  nw
502 implemented sequential for construct.
503 
504 Revision 1.7  2004/08/05 10:56:23  nw
505 implemented while loop construct
506 
507 Revision 1.6  2004/08/05 09:59:58  nw
508 implemented if statement
509 
510 Revision 1.5  2004/08/05 07:36:14  nw
511 made shell static for efficiency.
512 
513 Revision 1.4  2004/08/03 16:32:26  nw
514 remove unnecessary envId attrib from rules
515 implemented variable propagation into parameter values.
516 
517 Revision 1.3  2004/08/03 14:27:38  nw
518 added set/unset/scope features.
519 
520 Revision 1.2  2004/07/30 15:42:34  nw
521 merged in branch nww-itn06-bz#441 (groovy scripting)
522 
523 Revision 1.1.2.2  2004/07/30 15:10:04  nw
524 removed policy-based implementation,
525 adjusted tests, etc to use groovy implementation
526 
527 Revision 1.1.2.1  2004/07/30 14:00:10  nw
528 first working draft
529  
530 */