001/**
002 *
003 * Copyright (c) 2014, the Railo Company Ltd. All rights reserved.
004 *
005 * This library is free software; you can redistribute it and/or
006 * modify it under the terms of the GNU Lesser General Public
007 * License as published by the Free Software Foundation; either 
008 * version 2.1 of the License, or (at your option) any later version.
009 * 
010 * This library is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
013 * Lesser General Public License for more details.
014 * 
015 * You should have received a copy of the GNU Lesser General Public 
016 * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
017 * 
018 **/
019package lucee.runtime.engine;
020
021import java.io.IOException;
022import java.io.PrintWriter;
023import java.util.ArrayList;
024import java.util.Iterator;
025import java.util.List;
026import java.util.Map;
027
028import lucee.commons.io.IOUtil;
029import lucee.commons.io.SystemUtil;
030import lucee.commons.io.log.Log;
031import lucee.commons.io.log.LogUtil;
032import lucee.commons.io.res.Resource;
033import lucee.commons.io.res.filter.ExtensionResourceFilter;
034import lucee.commons.io.res.filter.ResourceFilter;
035import lucee.commons.io.res.util.ResourceUtil;
036import lucee.commons.lang.ExceptionUtil;
037import lucee.commons.lang.SystemOut;
038import lucee.commons.lang.types.RefBoolean;
039import lucee.runtime.CFMLFactoryImpl;
040import lucee.runtime.Mapping;
041import lucee.runtime.MappingImpl;
042import lucee.runtime.PageSource;
043import lucee.runtime.PageSourcePool;
044import lucee.runtime.config.ConfigImpl;
045import lucee.runtime.config.ConfigServer;
046import lucee.runtime.config.ConfigWeb;
047import lucee.runtime.config.ConfigWebAdmin;
048import lucee.runtime.config.ConfigWebImpl;
049import lucee.runtime.config.DeployHandler;
050import lucee.runtime.lock.LockManagerImpl;
051import lucee.runtime.net.smtp.SMTPConnectionPool;
052import lucee.runtime.op.Caster;
053import lucee.runtime.type.scope.ScopeContext;
054import lucee.runtime.type.scope.client.ClientFile;
055import lucee.runtime.type.util.ArrayUtil;
056
057/**
058 * own thread how check the main thread and his data 
059 */
060public final class Controler extends Thread {
061
062        private static final long TIMEOUT = 50*1000;
063        private int interval;
064        private long lastMinuteInterval=System.currentTimeMillis();
065        private long lastHourInterval=System.currentTimeMillis();
066        
067    private final Map contextes;
068    private final RefBoolean run;
069        //private ScheduleThread scheduleThread;
070        private final ConfigServer configServer;
071
072        /**
073         * @param contextes
074         * @param interval
075         * @param run 
076         */
077        public Controler(ConfigServer configServer,Map contextes,int interval, RefBoolean run) {                
078        this.contextes=contextes;
079                this.interval=interval;
080        this.run=run;
081        this.configServer=configServer;
082        
083        Runtime.getRuntime().addShutdownHook(new ShutdownHook(configServer));
084        
085        // Register Memory Notification Listener
086        //MemoryControler.init(configServer);
087        
088        }
089        
090        private static class ControlerThread extends Thread {
091                private Controler controler;
092                private CFMLFactoryImpl[] factories;
093                private boolean firstRun;
094                private long done=-1;
095                private Throwable t;
096                private Log log;
097                private long start;
098
099                public ControlerThread(Controler controler, CFMLFactoryImpl[] factories, boolean firstRun, Log log) {
100                        this.start=System.currentTimeMillis();
101                        this.controler=controler;
102                        this.factories=factories;
103                        this.firstRun=firstRun;
104                        this.log=log;
105                }
106
107                @Override
108                public void run() {
109                        long start=System.currentTimeMillis();
110                        try {
111                                controler.control(factories,firstRun);
112                                done=System.currentTimeMillis()-start;
113                        } 
114                        catch (Throwable t) {
115                ExceptionUtil.rethrowIfNecessary(t);
116                                this.t=t;
117                        }
118                        //long time=System.currentTimeMillis()-start;
119                        //if(time>10000) {
120                                //log.info("controller", "["+hashCode()+"] controller was running for "+time+"ms");
121                        //}
122                }
123        }
124        
125        
126        @Override
127        public void run() {
128                //scheduleThread.start();
129                boolean firstRun=true;
130                List<ControlerThread> threads=new ArrayList<ControlerThread>();
131                CFMLFactoryImpl factories[]=null;
132                while(run.toBooleanValue()) {
133                        // sleep
134                SystemUtil.sleep(interval);
135                
136            factories=toFactories(factories,contextes);
137            // start the thread that calls control
138            ControlerThread ct = new ControlerThread(this,factories,firstRun,configServer.getApplicationLogger());
139            ct.start();
140            threads.add(ct);
141            
142            if(threads.size()>10 && lastMinuteInterval+60000<System.currentTimeMillis())
143                configServer.getApplicationLogger().info("controller", threads.size()+" active controller threads");
144            
145            
146            // now we check all threads we have 
147            Iterator<ControlerThread> it = threads.iterator();
148            long time;
149            while(it.hasNext()){
150                ct=it.next();
151                //print.e(ct.hashCode());
152                time=System.currentTimeMillis()-ct.start;
153                // done
154                if(ct.done>=0) {
155                        if(time>10000) 
156                                configServer.getApplicationLogger().info("controller", "controler took "+ct.done+"ms to execute sucessfully.");
157                        it.remove();
158                }
159                // failed
160                else if(ct.t!=null){
161                        LogUtil.log(configServer.getApplicationLogger(), Log.LEVEL_ERROR, "controler", ct.t);
162                        it.remove();
163                }
164                // stop it!
165                else if(time>TIMEOUT) {
166                        SystemUtil.stop(ct,configServer.getApplicationLogger());
167                        //print.e(ct.getStackTrace());
168                        if(!ct.isAlive()) {
169                                configServer.getApplicationLogger().error("controller", "controler thread ["+ct.hashCode()+"] forced to stop after "+time+"ms");
170                                it.remove();
171                        }
172                        else {
173                                LogUtil.log(configServer.getApplicationLogger(), Log.LEVEL_ERROR, "controler","was not able to stop conroler thread running for "+time+"ms", ct.getStackTrace());
174                        }
175                }
176            }
177            
178
179            
180                if(factories.length>0) firstRun=false;
181            }    
182        }
183
184        
185        
186        
187        
188        private void control(CFMLFactoryImpl[] factories, boolean firstRun) {
189                long now = System.currentTimeMillis();
190        boolean doMinute=lastMinuteInterval+60000<now;
191        if(doMinute)lastMinuteInterval=now;
192        boolean doHour=(lastHourInterval+(1000*60*60))<now;
193        if(doHour)lastHourInterval=now;
194        
195        // broadcast cluster scope
196        try {
197                        ScopeContext.getClusterScope(configServer,true).broadcast();
198                } 
199        catch (Throwable t) {
200                ExceptionUtil.rethrowIfNecessary(t);
201                        t.printStackTrace();
202                }
203        
204
205        // every minute
206        if(doMinute) {
207                // deploy extensions, archives ...
208                        try{DeployHandler.deploy(configServer);}catch(Throwable t){
209                ExceptionUtil.rethrowIfNecessary(t);
210                t.printStackTrace();
211            }
212            try{ConfigWebAdmin.checkForChangesInConfigFile(configServer);}catch(Throwable t){
213                ExceptionUtil.rethrowIfNecessary(t);}
214        }
215        // every hour
216        if(doHour) {
217                try{configServer.checkPermGenSpace(true);}catch(Throwable t){
218                ExceptionUtil.rethrowIfNecessary(t);}
219        }
220        
221        for(int i=0;i<factories.length;i++) {
222            control(factories[i], doMinute, doHour,firstRun);
223        }
224        }
225
226
227        private void control(CFMLFactoryImpl cfmlFactory, boolean doMinute, boolean doHour, boolean firstRun) {
228                try {
229                                boolean isRunning=cfmlFactory.getUsedPageContextLength()>0;   
230                            if(isRunning) {
231                                        cfmlFactory.checkTimeout();
232                            }
233                                ConfigWeb config = null;
234                                
235                                if(firstRun) {
236                                        config = cfmlFactory.getConfig();
237                                        ThreadLocalConfig.register(config);
238                                        
239                                        config.reloadTimeServerOffset();
240                                        checkOldClientFile(config);
241                                        
242                                        //try{checkStorageScopeFile(config,Session.SCOPE_CLIENT);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
243                                        //try{checkStorageScopeFile(config,Session.SCOPE_SESSION);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
244                                        try{config.reloadTimeServerOffset();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
245                                        try{checkTempDirectorySize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
246                                        try{checkCacheFileSize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
247                                        try{cfmlFactory.getScopeContext().clearUnused();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
248                                }
249                                
250                                if(config==null) {
251                                        config = cfmlFactory.getConfig();
252                                }
253                                ThreadLocalConfig.register(config);
254                                
255                                //every Minute
256                                if(doMinute) {
257                                        if(config==null) {
258                                                config = cfmlFactory.getConfig();
259                                        }
260                                        ThreadLocalConfig.register(config);
261                                        
262                                        // double check templates
263                                        try{((ConfigWebImpl)config).getCompiler().checkWatched();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);t.printStackTrace();}
264
265                                        // deploy extensions, archives ...
266                                        try{DeployHandler.deploy(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);t.printStackTrace();}
267                                        
268                                        // clear unused DB Connections
269                                        try{((ConfigImpl)config).getDatasourceConnectionPool().clear();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
270                                        // clear all unused scopes
271                                        try{cfmlFactory.getScopeContext().clearUnused();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
272                                        // Memory usage
273                                        // clear Query Cache
274                                        /*try{
275                                                ConfigWebUtil.getCacheHandlerFactories(config).query.clean(null);
276                                                ConfigWebUtil.getCacheHandlerFactories(config).include.clean(null);
277                                                ConfigWebUtil.getCacheHandlerFactories(config).function.clean(null);
278                                                //cfmlFactory.getDefaultQueryCache().clearUnused(null);
279                                        }catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);t.printStackTrace();}*/
280                                        // contract Page Pool
281                                        //try{doClearPagePools((ConfigWebImpl) config);}catch(Throwable t){}
282                                        //try{checkPermGenSpace((ConfigWebImpl) config);}catch(Throwable t){}
283                                        try{doCheckMappings(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
284                                        try{doClearMailConnections();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
285                                        // clean LockManager
286                                        if(cfmlFactory.getUsedPageContextLength()==0)try{((LockManagerImpl)config.getLockManager()).clean();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
287                                        
288                                        try{ConfigWebAdmin.checkForChangesInConfigFile(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
289                        
290                                }
291                                // every hour
292                                if(doHour) {
293                                        if(config==null) {
294                                                config = cfmlFactory.getConfig();
295                                        }
296                                        ThreadLocalConfig.register(config);
297                                        
298                                        // time server offset
299                                        try{config.reloadTimeServerOffset();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
300                                        // check file based client/session scope
301                                        //try{checkStorageScopeFile(config,Session.SCOPE_CLIENT);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
302                                        //try{checkStorageScopeFile(config,Session.SCOPE_SESSION);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
303                                        // check temp directory
304                                        try{checkTempDirectorySize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
305                                        // check cache directory
306                                        try{checkCacheFileSize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
307                                }
308
309                        try{configServer.checkPermGenSpace(true);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);}
310                        }
311                        catch(Throwable t){
312                                ExceptionUtil.rethrowIfNecessary(t);
313                        }
314                        finally{
315                                ThreadLocalConfig.release();
316                        }
317        }
318
319        private CFMLFactoryImpl[] toFactories(CFMLFactoryImpl[] factories,Map contextes) {
320                if(factories==null || factories.length!=contextes.size())
321                        factories=(CFMLFactoryImpl[]) contextes.values().toArray(new CFMLFactoryImpl[contextes.size()]);
322                
323                return factories;
324        }
325
326        private void doClearMailConnections() {
327                SMTPConnectionPool.closeSessions();
328        }
329
330        private void checkOldClientFile(ConfigWeb config) {
331                ExtensionResourceFilter filter = new ExtensionResourceFilter(".script",false);
332                
333                // move old structured file in new structure
334                try {
335                        Resource dir = config.getClientScopeDir(),trgres;
336                        Resource[] children = dir.listResources(filter);
337                        String src,trg;
338                        int index;
339                        for(int i=0;i<children.length;i++) {
340                                src=children[i].getName();
341                                index=src.indexOf('-');
342
343                                trg=ClientFile.getFolderName(src.substring(0,index), src.substring(index+1),false);
344                                trgres=dir.getRealResource(trg);
345                                if(!trgres.exists()){
346                                        trgres.createFile(true);
347                                        ResourceUtil.copy(children[i],trgres);
348                                }
349                                        //children[i].moveTo(trgres);
350                                children[i].delete();
351                                
352                        }
353                } catch (Throwable t) {ExceptionUtil.rethrowIfNecessary(t);}
354        }
355
356        private void checkCacheFileSize(ConfigWeb config) {
357                checkSize(config,config.getCacheDir(),config.getCacheDirSize(),new ExtensionResourceFilter(".cache"));
358        }
359        
360        private void checkTempDirectorySize(ConfigWeb config) {
361                checkSize(config,config.getTempDirectory(),1024*1024*1024,null);
362        }
363        
364        private void checkSize(ConfigWeb config,Resource dir,long maxSize, ResourceFilter filter) {
365                if(!dir.exists()) return;
366                Resource res=null;
367                int count=ArrayUtil.size(filter==null?dir.list():dir.list(filter));
368                long size=ResourceUtil.getRealSize(dir,filter);
369                PrintWriter out = config.getOutWriter();
370                SystemOut.printDate(out,"check size of directory ["+dir+"]");
371                SystemOut.printDate(out,"- current size ["+size+"]");
372                SystemOut.printDate(out,"- max size     ["+maxSize+"]");
373                int len=-1;
374                while(count>100000 || size>maxSize) {
375                        Resource[] files = filter==null?dir.listResources():dir.listResources(filter);
376                        if(len==files.length) break;// protect from inifinti loop
377                        len=files.length;
378                        for(int i=0;i<files.length;i++) {
379                                if(res==null || res.lastModified()>files[i].lastModified()) {
380                                        res=files[i];
381                                }
382                        }
383                        if(res!=null) {
384                                size-=res.length();
385                                try {
386                                        res.remove(true);
387                                        count--;
388                                } catch (IOException e) {
389                                        SystemOut.printDate(out,"cannot remove resource "+res.getAbsolutePath());
390                                        break;
391                                }
392                        }
393                        res=null;
394                }
395                
396        }
397
398        private void doCheckMappings(ConfigWeb config) {
399        Mapping[] mappings = config.getMappings();
400        for(int i=0;i<mappings.length;i++) {
401            Mapping mapping = mappings[i];
402            mapping.check();
403        }
404    }
405
406        private PageSourcePool[] getPageSourcePools(ConfigWeb config) {
407                return getPageSourcePools(config.getMappings());
408        }
409
410        private PageSourcePool[] getPageSourcePools(Mapping... mappings) {
411        PageSourcePool[] pools=new PageSourcePool[mappings.length];
412        //int size=0;
413        
414        for(int i=0;i<mappings.length;i++) {
415            pools[i]=((MappingImpl)mappings[i]).getPageSourcePool();
416            //size+=pools[i].size();
417        }
418        return pools;
419    }
420    private int getPageSourcePoolSize(PageSourcePool[] pools) {
421        int size=0;
422        for(int i=0;i<pools.length;i++)size+=pools[i].size();
423        return size;
424    }
425    private void removeOldest(PageSourcePool[] pools) {
426        PageSourcePool pool=null;
427        Object key=null;
428        PageSource ps=null;
429        
430        long date=-1;
431        for(int i=0;i<pools.length;i++) {
432                try {
433                    Object[] keys=pools[i].keys();
434                    for(int y=0;y<keys.length;y++) {
435                        ps = pools[i].getPageSource(keys[y],false);
436                        if(date==-1 || date>ps.getLastAccessTime()) {
437                            pool=pools[i];
438                            key=keys[y];
439                            date=ps.getLastAccessTime();
440                        }
441                    }
442                }
443                catch(Throwable t) {
444                        ExceptionUtil.rethrowIfNecessary(t);
445                        pools[i].clear();
446                }
447                
448        }
449        if(pool!=null)pool.remove(key);
450    }
451    private void clear(PageSourcePool[] pools) {
452        for(int i=0;i<pools.length;i++) {
453                pools[i].clear();
454        }
455    }
456
457    /*private void doLogMemoryUsage(ConfigWeb config) {
458                if(config.logMemoryUsage()&& config.getMemoryLogger()!=null)
459                        config.getMemoryLogger().write();
460        }*/
461    
462    
463    static class ExpiresFilter implements ResourceFilter {
464
465                private long time;
466                private boolean allowDir;
467
468                public ExpiresFilter(long time, boolean allowDir) {
469                        this.allowDir=allowDir;
470                        this.time=time;
471                }
472
473                public boolean accept(Resource res) {
474
475                        if(res.isDirectory()) return allowDir;
476                        
477                        // load content
478                        String str=null;
479                        try {
480                                str = IOUtil.toString(res,"UTF-8");
481                        } 
482                        catch (IOException e) {
483                                return false;
484                        }
485                        
486                        int index=str.indexOf(':');
487                        if(index!=-1){
488                                long expires=Caster.toLongValue(str.substring(0,index),-1L);
489                                // check is for backward compatibility, old files have no expires date inside. they do ot expire
490                                if(expires!=-1) {
491                                        if(expires<System.currentTimeMillis()){
492                                                return true;
493                                        }
494                                        str=str.substring(index+1);
495                                        return false;
496                                }
497                        }
498                        // old files not having a timestamp inside
499                        else if(res.lastModified()<=time) {
500                                return true;
501                                
502                        }
503                        return false;
504                }
505        
506    }
507}