001/** 002 * 003 * Copyright (c) 2014, the Railo Company Ltd. All rights reserved. 004 * 005 * This library is free software; you can redistribute it and/or 006 * modify it under the terms of the GNU Lesser General Public 007 * License as published by the Free Software Foundation; either 008 * version 2.1 of the License, or (at your option) any later version. 009 * 010 * This library is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 013 * Lesser General Public License for more details. 014 * 015 * You should have received a copy of the GNU Lesser General Public 016 * License along with this library. If not, see <http://www.gnu.org/licenses/>. 017 * 018 **/ 019package lucee.runtime.engine; 020 021import java.io.IOException; 022import java.io.PrintWriter; 023import java.util.ArrayList; 024import java.util.Iterator; 025import java.util.List; 026import java.util.Map; 027 028import lucee.commons.io.IOUtil; 029import lucee.commons.io.SystemUtil; 030import lucee.commons.io.log.Log; 031import lucee.commons.io.log.LogUtil; 032import lucee.commons.io.res.Resource; 033import lucee.commons.io.res.filter.ExtensionResourceFilter; 034import lucee.commons.io.res.filter.ResourceFilter; 035import lucee.commons.io.res.util.ResourceUtil; 036import lucee.commons.lang.ExceptionUtil; 037import lucee.commons.lang.SystemOut; 038import lucee.commons.lang.types.RefBoolean; 039import lucee.runtime.CFMLFactoryImpl; 040import lucee.runtime.Mapping; 041import lucee.runtime.MappingImpl; 042import lucee.runtime.PageSource; 043import lucee.runtime.PageSourcePool; 044import lucee.runtime.config.ConfigImpl; 045import lucee.runtime.config.ConfigServer; 046import lucee.runtime.config.ConfigWeb; 047import lucee.runtime.config.ConfigWebAdmin; 048import lucee.runtime.config.ConfigWebImpl; 049import lucee.runtime.config.DeployHandler; 050import lucee.runtime.lock.LockManagerImpl; 051import lucee.runtime.net.smtp.SMTPConnectionPool; 052import lucee.runtime.op.Caster; 053import lucee.runtime.type.scope.ScopeContext; 054import lucee.runtime.type.scope.client.ClientFile; 055import lucee.runtime.type.util.ArrayUtil; 056 057/** 058 * own thread how check the main thread and his data 059 */ 060public final class Controler extends Thread { 061 062 private static final long TIMEOUT = 50*1000; 063 private int interval; 064 private long lastMinuteInterval=System.currentTimeMillis(); 065 private long lastHourInterval=System.currentTimeMillis(); 066 067 private final Map contextes; 068 private final RefBoolean run; 069 //private ScheduleThread scheduleThread; 070 private final ConfigServer configServer; 071 072 /** 073 * @param contextes 074 * @param interval 075 * @param run 076 */ 077 public Controler(ConfigServer configServer,Map contextes,int interval, RefBoolean run) { 078 this.contextes=contextes; 079 this.interval=interval; 080 this.run=run; 081 this.configServer=configServer; 082 083 Runtime.getRuntime().addShutdownHook(new ShutdownHook(configServer)); 084 085 // Register Memory Notification Listener 086 //MemoryControler.init(configServer); 087 088 } 089 090 private static class ControlerThread extends Thread { 091 private Controler controler; 092 private CFMLFactoryImpl[] factories; 093 private boolean firstRun; 094 private long done=-1; 095 private Throwable t; 096 private Log log; 097 private long start; 098 099 public ControlerThread(Controler controler, CFMLFactoryImpl[] factories, boolean firstRun, Log log) { 100 this.start=System.currentTimeMillis(); 101 this.controler=controler; 102 this.factories=factories; 103 this.firstRun=firstRun; 104 this.log=log; 105 } 106 107 @Override 108 public void run() { 109 long start=System.currentTimeMillis(); 110 try { 111 controler.control(factories,firstRun); 112 done=System.currentTimeMillis()-start; 113 } 114 catch (Throwable t) { 115 ExceptionUtil.rethrowIfNecessary(t); 116 this.t=t; 117 } 118 //long time=System.currentTimeMillis()-start; 119 //if(time>10000) { 120 //log.info("controller", "["+hashCode()+"] controller was running for "+time+"ms"); 121 //} 122 } 123 } 124 125 126 @Override 127 public void run() { 128 //scheduleThread.start(); 129 boolean firstRun=true; 130 List<ControlerThread> threads=new ArrayList<ControlerThread>(); 131 CFMLFactoryImpl factories[]=null; 132 while(run.toBooleanValue()) { 133 // sleep 134 SystemUtil.sleep(interval); 135 136 factories=toFactories(factories,contextes); 137 // start the thread that calls control 138 ControlerThread ct = new ControlerThread(this,factories,firstRun,configServer.getApplicationLogger()); 139 ct.start(); 140 threads.add(ct); 141 142 if(threads.size()>10 && lastMinuteInterval+60000<System.currentTimeMillis()) 143 configServer.getApplicationLogger().info("controller", threads.size()+" active controller threads"); 144 145 146 // now we check all threads we have 147 Iterator<ControlerThread> it = threads.iterator(); 148 long time; 149 while(it.hasNext()){ 150 ct=it.next(); 151 //print.e(ct.hashCode()); 152 time=System.currentTimeMillis()-ct.start; 153 // done 154 if(ct.done>=0) { 155 if(time>10000) 156 configServer.getApplicationLogger().info("controller", "controler took "+ct.done+"ms to execute sucessfully."); 157 it.remove(); 158 } 159 // failed 160 else if(ct.t!=null){ 161 LogUtil.log(configServer.getApplicationLogger(), Log.LEVEL_ERROR, "controler", ct.t); 162 it.remove(); 163 } 164 // stop it! 165 else if(time>TIMEOUT) { 166 SystemUtil.stop(ct,configServer.getApplicationLogger()); 167 //print.e(ct.getStackTrace()); 168 if(!ct.isAlive()) { 169 configServer.getApplicationLogger().error("controller", "controler thread ["+ct.hashCode()+"] forced to stop after "+time+"ms"); 170 it.remove(); 171 } 172 else { 173 LogUtil.log(configServer.getApplicationLogger(), Log.LEVEL_ERROR, "controler","was not able to stop conroler thread running for "+time+"ms", ct.getStackTrace()); 174 } 175 } 176 } 177 178 179 180 if(factories.length>0) firstRun=false; 181 } 182 } 183 184 185 186 187 188 private void control(CFMLFactoryImpl[] factories, boolean firstRun) { 189 long now = System.currentTimeMillis(); 190 boolean doMinute=lastMinuteInterval+60000<now; 191 if(doMinute)lastMinuteInterval=now; 192 boolean doHour=(lastHourInterval+(1000*60*60))<now; 193 if(doHour)lastHourInterval=now; 194 195 // broadcast cluster scope 196 try { 197 ScopeContext.getClusterScope(configServer,true).broadcast(); 198 } 199 catch (Throwable t) { 200 ExceptionUtil.rethrowIfNecessary(t); 201 t.printStackTrace(); 202 } 203 204 205 // every minute 206 if(doMinute) { 207 // deploy extensions, archives ... 208 try{DeployHandler.deploy(configServer);}catch(Throwable t){ 209 ExceptionUtil.rethrowIfNecessary(t); 210 t.printStackTrace(); 211 } 212 try{ConfigWebAdmin.checkForChangesInConfigFile(configServer);}catch(Throwable t){ 213 ExceptionUtil.rethrowIfNecessary(t);} 214 } 215 // every hour 216 if(doHour) { 217 try{configServer.checkPermGenSpace(true);}catch(Throwable t){ 218 ExceptionUtil.rethrowIfNecessary(t);} 219 } 220 221 for(int i=0;i<factories.length;i++) { 222 control(factories[i], doMinute, doHour,firstRun); 223 } 224 } 225 226 227 private void control(CFMLFactoryImpl cfmlFactory, boolean doMinute, boolean doHour, boolean firstRun) { 228 try { 229 boolean isRunning=cfmlFactory.getUsedPageContextLength()>0; 230 if(isRunning) { 231 cfmlFactory.checkTimeout(); 232 } 233 ConfigWeb config = null; 234 235 if(firstRun) { 236 config = cfmlFactory.getConfig(); 237 ThreadLocalConfig.register(config); 238 239 config.reloadTimeServerOffset(); 240 checkOldClientFile(config); 241 242 //try{checkStorageScopeFile(config,Session.SCOPE_CLIENT);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 243 //try{checkStorageScopeFile(config,Session.SCOPE_SESSION);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 244 try{config.reloadTimeServerOffset();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 245 try{checkTempDirectorySize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 246 try{checkCacheFileSize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 247 try{cfmlFactory.getScopeContext().clearUnused();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 248 } 249 250 if(config==null) { 251 config = cfmlFactory.getConfig(); 252 } 253 ThreadLocalConfig.register(config); 254 255 //every Minute 256 if(doMinute) { 257 if(config==null) { 258 config = cfmlFactory.getConfig(); 259 } 260 ThreadLocalConfig.register(config); 261 262 // double check templates 263 try{((ConfigWebImpl)config).getCompiler().checkWatched();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);t.printStackTrace();} 264 265 // deploy extensions, archives ... 266 try{DeployHandler.deploy(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);t.printStackTrace();} 267 268 // clear unused DB Connections 269 try{((ConfigImpl)config).getDatasourceConnectionPool().clear();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 270 // clear all unused scopes 271 try{cfmlFactory.getScopeContext().clearUnused();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 272 // Memory usage 273 // clear Query Cache 274 /*try{ 275 ConfigWebUtil.getCacheHandlerFactories(config).query.clean(null); 276 ConfigWebUtil.getCacheHandlerFactories(config).include.clean(null); 277 ConfigWebUtil.getCacheHandlerFactories(config).function.clean(null); 278 //cfmlFactory.getDefaultQueryCache().clearUnused(null); 279 }catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);t.printStackTrace();}*/ 280 // contract Page Pool 281 //try{doClearPagePools((ConfigWebImpl) config);}catch(Throwable t){} 282 //try{checkPermGenSpace((ConfigWebImpl) config);}catch(Throwable t){} 283 try{doCheckMappings(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 284 try{doClearMailConnections();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 285 // clean LockManager 286 if(cfmlFactory.getUsedPageContextLength()==0)try{((LockManagerImpl)config.getLockManager()).clean();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 287 288 try{ConfigWebAdmin.checkForChangesInConfigFile(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 289 290 } 291 // every hour 292 if(doHour) { 293 if(config==null) { 294 config = cfmlFactory.getConfig(); 295 } 296 ThreadLocalConfig.register(config); 297 298 // time server offset 299 try{config.reloadTimeServerOffset();}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 300 // check file based client/session scope 301 //try{checkStorageScopeFile(config,Session.SCOPE_CLIENT);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 302 //try{checkStorageScopeFile(config,Session.SCOPE_SESSION);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 303 // check temp directory 304 try{checkTempDirectorySize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 305 // check cache directory 306 try{checkCacheFileSize(config);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 307 } 308 309 try{configServer.checkPermGenSpace(true);}catch(Throwable t){ExceptionUtil.rethrowIfNecessary(t);} 310 } 311 catch(Throwable t){ 312 ExceptionUtil.rethrowIfNecessary(t); 313 } 314 finally{ 315 ThreadLocalConfig.release(); 316 } 317 } 318 319 private CFMLFactoryImpl[] toFactories(CFMLFactoryImpl[] factories,Map contextes) { 320 if(factories==null || factories.length!=contextes.size()) 321 factories=(CFMLFactoryImpl[]) contextes.values().toArray(new CFMLFactoryImpl[contextes.size()]); 322 323 return factories; 324 } 325 326 private void doClearMailConnections() { 327 SMTPConnectionPool.closeSessions(); 328 } 329 330 private void checkOldClientFile(ConfigWeb config) { 331 ExtensionResourceFilter filter = new ExtensionResourceFilter(".script",false); 332 333 // move old structured file in new structure 334 try { 335 Resource dir = config.getClientScopeDir(),trgres; 336 Resource[] children = dir.listResources(filter); 337 String src,trg; 338 int index; 339 for(int i=0;i<children.length;i++) { 340 src=children[i].getName(); 341 index=src.indexOf('-'); 342 343 trg=ClientFile.getFolderName(src.substring(0,index), src.substring(index+1),false); 344 trgres=dir.getRealResource(trg); 345 if(!trgres.exists()){ 346 trgres.createFile(true); 347 ResourceUtil.copy(children[i],trgres); 348 } 349 //children[i].moveTo(trgres); 350 children[i].delete(); 351 352 } 353 } catch (Throwable t) {ExceptionUtil.rethrowIfNecessary(t);} 354 } 355 356 private void checkCacheFileSize(ConfigWeb config) { 357 checkSize(config,config.getCacheDir(),config.getCacheDirSize(),new ExtensionResourceFilter(".cache")); 358 } 359 360 private void checkTempDirectorySize(ConfigWeb config) { 361 checkSize(config,config.getTempDirectory(),1024*1024*1024,null); 362 } 363 364 private void checkSize(ConfigWeb config,Resource dir,long maxSize, ResourceFilter filter) { 365 if(!dir.exists()) return; 366 Resource res=null; 367 int count=ArrayUtil.size(filter==null?dir.list():dir.list(filter)); 368 long size=ResourceUtil.getRealSize(dir,filter); 369 PrintWriter out = config.getOutWriter(); 370 SystemOut.printDate(out,"check size of directory ["+dir+"]"); 371 SystemOut.printDate(out,"- current size ["+size+"]"); 372 SystemOut.printDate(out,"- max size ["+maxSize+"]"); 373 int len=-1; 374 while(count>100000 || size>maxSize) { 375 Resource[] files = filter==null?dir.listResources():dir.listResources(filter); 376 if(len==files.length) break;// protect from inifinti loop 377 len=files.length; 378 for(int i=0;i<files.length;i++) { 379 if(res==null || res.lastModified()>files[i].lastModified()) { 380 res=files[i]; 381 } 382 } 383 if(res!=null) { 384 size-=res.length(); 385 try { 386 res.remove(true); 387 count--; 388 } catch (IOException e) { 389 SystemOut.printDate(out,"cannot remove resource "+res.getAbsolutePath()); 390 break; 391 } 392 } 393 res=null; 394 } 395 396 } 397 398 private void doCheckMappings(ConfigWeb config) { 399 Mapping[] mappings = config.getMappings(); 400 for(int i=0;i<mappings.length;i++) { 401 Mapping mapping = mappings[i]; 402 mapping.check(); 403 } 404 } 405 406 private PageSourcePool[] getPageSourcePools(ConfigWeb config) { 407 return getPageSourcePools(config.getMappings()); 408 } 409 410 private PageSourcePool[] getPageSourcePools(Mapping... mappings) { 411 PageSourcePool[] pools=new PageSourcePool[mappings.length]; 412 //int size=0; 413 414 for(int i=0;i<mappings.length;i++) { 415 pools[i]=((MappingImpl)mappings[i]).getPageSourcePool(); 416 //size+=pools[i].size(); 417 } 418 return pools; 419 } 420 private int getPageSourcePoolSize(PageSourcePool[] pools) { 421 int size=0; 422 for(int i=0;i<pools.length;i++)size+=pools[i].size(); 423 return size; 424 } 425 private void removeOldest(PageSourcePool[] pools) { 426 PageSourcePool pool=null; 427 Object key=null; 428 PageSource ps=null; 429 430 long date=-1; 431 for(int i=0;i<pools.length;i++) { 432 try { 433 Object[] keys=pools[i].keys(); 434 for(int y=0;y<keys.length;y++) { 435 ps = pools[i].getPageSource(keys[y],false); 436 if(date==-1 || date>ps.getLastAccessTime()) { 437 pool=pools[i]; 438 key=keys[y]; 439 date=ps.getLastAccessTime(); 440 } 441 } 442 } 443 catch(Throwable t) { 444 ExceptionUtil.rethrowIfNecessary(t); 445 pools[i].clear(); 446 } 447 448 } 449 if(pool!=null)pool.remove(key); 450 } 451 private void clear(PageSourcePool[] pools) { 452 for(int i=0;i<pools.length;i++) { 453 pools[i].clear(); 454 } 455 } 456 457 /*private void doLogMemoryUsage(ConfigWeb config) { 458 if(config.logMemoryUsage()&& config.getMemoryLogger()!=null) 459 config.getMemoryLogger().write(); 460 }*/ 461 462 463 static class ExpiresFilter implements ResourceFilter { 464 465 private long time; 466 private boolean allowDir; 467 468 public ExpiresFilter(long time, boolean allowDir) { 469 this.allowDir=allowDir; 470 this.time=time; 471 } 472 473 public boolean accept(Resource res) { 474 475 if(res.isDirectory()) return allowDir; 476 477 // load content 478 String str=null; 479 try { 480 str = IOUtil.toString(res,"UTF-8"); 481 } 482 catch (IOException e) { 483 return false; 484 } 485 486 int index=str.indexOf(':'); 487 if(index!=-1){ 488 long expires=Caster.toLongValue(str.substring(0,index),-1L); 489 // check is for backward compatibility, old files have no expires date inside. they do ot expire 490 if(expires!=-1) { 491 if(expires<System.currentTimeMillis()){ 492 return true; 493 } 494 str=str.substring(index+1); 495 return false; 496 } 497 } 498 // old files not having a timestamp inside 499 else if(res.lastModified()<=time) { 500 return true; 501 502 } 503 return false; 504 } 505 506 } 507}