001/**
002 *
003 * Copyright (c) 2014, the Railo Company Ltd. All rights reserved.
004 *
005 * This library is free software; you can redistribute it and/or
006 * modify it under the terms of the GNU Lesser General Public
007 * License as published by the Free Software Foundation; either 
008 * version 2.1 of the License, or (at your option) any later version.
009 * 
010 * This library is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
013 * Lesser General Public License for more details.
014 * 
015 * You should have received a copy of the GNU Lesser General Public 
016 * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
017 * 
018 **/
019package lucee.runtime.search;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024import java.util.Iterator;
025
026import lucee.commons.io.IOUtil;
027import lucee.commons.io.log.LogAndSource;
028import lucee.commons.io.res.Resource;
029import lucee.commons.io.res.ResourceProvider;
030import lucee.commons.io.res.ResourcesImpl;
031import lucee.commons.lang.StringUtil;
032import lucee.runtime.Info;
033import lucee.runtime.config.Config;
034import lucee.runtime.engine.ThreadLocalPageContext;
035import lucee.runtime.exp.DatabaseException;
036import lucee.runtime.exp.PageException;
037import lucee.runtime.op.date.DateCaster;
038import lucee.runtime.type.Collection.Key;
039import lucee.runtime.type.KeyImpl;
040import lucee.runtime.type.Query;
041import lucee.runtime.type.QueryImpl;
042import lucee.runtime.type.Struct;
043import lucee.runtime.type.StructImpl;
044import lucee.runtime.type.dt.DateTime;
045import lucee.runtime.type.dt.DateTimeImpl;
046import lucee.runtime.type.util.KeyConstants;
047import lucee.runtime.type.util.ListUtil;
048
049import org.apache.xerces.parsers.DOMParser;
050import org.apache.xml.serialize.OutputFormat;
051import org.apache.xml.serialize.XMLSerializer;
052import org.w3c.dom.Document;
053import org.w3c.dom.Element;
054import org.w3c.dom.Node;
055import org.w3c.dom.NodeList;
056import org.xml.sax.InputSource;
057import org.xml.sax.SAXException;
058
059/**
060 * 
061 */
062public abstract class SearchEngineSupport implements SearchEngine {
063    
064        private Resource searchFile;
065        private Resource searchDir;
066        private Document doc;
067    Struct collections=new StructImpl();
068        protected Config config;
069        
070        @Override
071        public void init(lucee.runtime.config.Config config,Resource searchDir, 
072                        LogAndSource log/* always null*/) throws SAXException, IOException, SearchException {
073                this.config=config;
074                this.searchDir=searchDir;
075                this.searchFile=searchDir.getRealResource("search.xml");
076                if(!searchFile.exists()) createSearchFile(searchFile);
077                
078                DOMParser parser = new DOMParser();
079                InputStream is=null;
080            try {
081                        is = IOUtil.toBufferedInputStream(searchFile.getInputStream());
082                InputSource source = new InputSource(is);
083                parser.parse(source);
084            }
085            finally {
086                IOUtil.closeEL(is);
087            }
088        doc = parser.getDocument();
089                
090        
091        readCollections(config);
092        } 
093        
094        @Override
095        public final SearchCollection getCollectionByName(String name) throws SearchException {
096                Object o=collections.get(name.toLowerCase(),null);
097                if(o!=null)return (SearchCollection) o; 
098                throw new SearchException("collection "+name+" is undefined");
099        }
100        
101        @Override
102        public final Query getCollectionsAsQuery() {
103        final String v="VARCHAR";
104        Query query=null;
105        String[] cols = new String[]{"external","language","mapped","name","online","path","registered","lastmodified","categories","charset","created",
106                                                                 "size","doccount"};
107        String[] types = new String[]{"BOOLEAN",v,"BOOLEAN",v,"BOOLEAN",v,v,"DATE","BOOLEAN",v,"OBJECT","DOUBLE","DOUBLE"};
108        try {
109            query=new QueryImpl(cols,types, collections.size(),"query");
110        } catch (DatabaseException e) {
111            query=new QueryImpl(cols, collections.size(),"query");
112        }
113        
114        //Collection.Key[] keys = collections.keys();
115            Iterator<Object> it = collections.valueIterator();
116            int i=-1;
117        while(it.hasNext()) {
118                i++;
119                try {
120                        SearchCollection coll = (SearchCollection) it.next();
121                query.setAt(KeyConstants._external,i+1,Boolean.FALSE);
122                query.setAt(KeyConstants._charset,i+1,"UTF-8");
123                query.setAt(KeyConstants._created,i+1,coll.created());
124                
125                query.setAt("categories",i+1,Boolean.TRUE);
126                        query.setAt(KeyConstants._language,i+1,coll.getLanguage());
127                        query.setAt("mapped",i+1,Boolean.FALSE);
128                        query.setAt(KeyConstants._name,i+1,coll.getName());
129                        query.setAt(KeyConstants._online,i+1,Boolean.TRUE);
130                        query.setAt(KeyConstants._path,i+1,coll.getPath().getAbsolutePath());
131                        query.setAt("registered",i+1,"CF");
132                        query.setAt(KeyConstants._lastmodified,i+1,coll.getLastUpdate());
133                        query.setAt(KeyConstants._size,i+1,new Double(coll.getSize()));
134                        query.setAt("doccount",i+1,new Double(coll.getDocumentCount())); 
135                }
136                    catch(PageException pe) {}
137            }
138                return query;
139        }
140        
141        @Override
142        public final SearchCollection createCollection(String name,Resource path, String language, boolean allowOverwrite) throws SearchException {
143            SearchCollection coll = _createCollection(name,path,language);
144            coll.create();
145            addCollection(coll,allowOverwrite);
146            return coll;
147        }
148        
149        /**
150         * Creates a new Collection, will be invoked by createCollection
151         * @param name The Name of the Collection
152         * @param path the path to store
153         * @param language The language of the collection
154         * @return New SearchCollection
155         * @throws SearchException
156         */
157        protected abstract SearchCollection _createCollection(String name,Resource path, String language) throws SearchException;
158        
159        /**
160         * adds a new Collection to the storage
161         * @param collection
162         * @param allowOverwrite if allowOverwrite is false and a collection already exist -> throw Exception
163         * @throws SearchException
164         */
165        private final synchronized void addCollection(SearchCollection collection, boolean allowOverwrite) throws SearchException {
166            Object o = collections.get(collection.getName(),null);
167            if(!allowOverwrite && o!=null)
168                    throw new SearchException("there is already a collection with name "+collection.getName());
169                collections.setEL(collection.getName(),collection);
170                // update
171                if(o!=null) {
172                    setAttributes(getCollectionElement(collection.getName()),collection);
173                }
174                // create
175                else {
176                    doc.getDocumentElement().appendChild(toElement(collection));
177                }
178                store();
179        }
180
181        /**
182         * removes a Collection from the storage
183         * @param collection Collection to remove
184         * @throws SearchException
185         */
186        protected final synchronized void removeCollection(SearchCollection collection)
187                        throws SearchException {
188            removeCollection(collection.getName());
189            _removeCollection(collection);
190        }
191    
192    /**
193     * removes a Collection from the storage
194     * @param collection Collection to remove
195     * @throws SearchException
196     */
197    protected abstract void _removeCollection(SearchCollection collection) throws SearchException;
198    
199    /**
200     * removes a Collection from the storage
201     * @param name Name of the Collection to remove
202     * @throws SearchException
203     */
204    private final synchronized void removeCollection(String name) throws SearchException {
205        try {
206            collections.remove(KeyImpl.init(name));
207            doc.getDocumentElement().removeChild(getCollectionElement(name));
208            store();
209        } 
210        catch (PageException e) {
211            throw new SearchException("can't remove collection "+name+", collection doesn't exist");
212        }
213    }
214    
215    
216    /**
217     * purge a Collection 
218     * @param collection Collection to purge
219     * @throws SearchException
220     */
221    protected final synchronized void purgeCollection(SearchCollection collection)
222            throws SearchException {
223        
224        purgeCollection(collection.getName());
225    }
226    
227    /**
228     * purge a Collection
229     * @param name Name of the Collection to purge
230     * @throws SearchException
231     */
232    private final synchronized void purgeCollection(String name) throws SearchException {
233        
234            //Map map=(Map)collections.get(name);
235            //if(map!=null)map.clear();
236            Element parent = getCollectionElement(name);
237            NodeList list=parent.getChildNodes();
238            int len=list.getLength();
239            for(int i=len-1;i>=0;i--) {
240                parent.removeChild(list.item(i));
241            }
242            //doc.getDocumentElement().removeChild(getCollectionElement(name));
243            store();
244        
245    }
246
247        @Override
248        public Resource getDirectory() {
249                return searchDir;
250        }
251
252        @Override
253        public LogAndSource getLogger() {
254        throw new RuntimeException("this method is no longer supported, call instead Config.getLogger");
255        }
256
257    /**
258     * return XML Element matching collection name
259     * @param name
260     * @return matching XML Element
261     */
262    protected final Element getCollectionElement(String name) {
263        Element root = doc.getDocumentElement();
264        NodeList children = root.getChildNodes();
265        int len=children.getLength();
266        for(int i=0;i<len;i++) {
267            Node n=children.item(i);
268            if(n instanceof Element && n.getNodeName().equals("collection")) {
269                Element el = (Element)n;
270                if(el.getAttribute("name").equalsIgnoreCase(name)) return el;
271            }
272        }
273        return null;
274    }
275
276    @Override
277    public Element getIndexElement(Element collElement, String id) {
278        
279        NodeList children = collElement.getChildNodes();
280        int len=children.getLength();
281        for(int i=0;i<len;i++) {
282            Node n=children.item(i);
283            if(n instanceof Element && n.getNodeName().equals("index")) {
284                Element el = (Element)n;
285                if(el.getAttribute("id").equals(id)) return el;
286            }
287        }
288        return null;
289    }
290
291    /**
292     * translate a collection object to a XML Element
293     * @param coll Collection to translate
294     * @return XML Element
295     */
296    private final Element toElement(SearchCollection coll) {
297        Element el = doc.createElement("collection");
298        setAttributes(el,coll);   
299        return el;
300    }
301
302    /**
303     * translate a collection object to a XML Element
304     * @param index Index to translate
305     * @return XML Element
306     * @throws SearchException
307     */
308    protected final Element toElement(SearchIndex index) throws SearchException {
309        Element el = doc.createElement("index");
310        setAttributes(el,index);   
311        return el;
312    }
313    
314    /**
315     * sets all attributes in XML Element from Search Collection
316     * @param el
317     * @param coll
318     */
319    private final void setAttributes(Element el,SearchCollection coll) {
320        if(el==null) return;
321        setAttribute(el,"language",coll.getLanguage());
322        setAttribute(el,"name",coll.getName());
323        
324        String value = coll.getLastUpdate().castToString(null);
325        if(value!=null)setAttribute(el,"lastUpdate",value);
326        value=coll.getCreated().castToString(null);
327        if(value!=null)setAttribute(el,"created",value);
328        
329        setAttribute(el,"path",coll.getPath().getAbsolutePath());
330    }
331    
332    /**
333     * sets all attributes in XML Element from Search Index
334     * @param el
335     * @param index
336     * @throws SearchException
337     */
338    protected final void setAttributes(Element el,SearchIndex index) throws SearchException {
339        if(el==null) return;
340        setAttribute(el,"categoryTree",index.getCategoryTree());
341        setAttribute(el,"category",ListUtil.arrayToList(index.getCategories(),","));
342        setAttribute(el,"custom1",index.getCustom1());
343        setAttribute(el,"custom2",index.getCustom2());
344        setAttribute(el,"custom3",index.getCustom3());
345        setAttribute(el,"custom4",index.getCustom4());
346        setAttribute(el,"id",index.getId());
347        setAttribute(el,"key",index.getKey());
348        setAttribute(el,"language",index.getLanguage());
349        setAttribute(el,"title",index.getTitle());
350        setAttribute(el,"extensions",ListUtil.arrayToList(index.getExtensions(),","));
351        setAttribute(el,"type",SearchIndex.toStringType(index.getType()));
352        setAttribute(el,"urlpath",index.getUrlpath());
353        setAttribute(el,"query",index.getQuery());
354    }
355
356        /**
357         * helper method to set a attribute
358     * @param el
359     * @param name
360     * @param value
361     */
362    private void setAttribute(Element el, String name, String value) {
363        if(value!=null)el.setAttribute(name,value);
364    }
365
366    /**
367         * read in collections
368     * @param config 
369         * @throws SearchException
370     */
371    private void readCollections(Config config) throws SearchException {
372        Element root = doc.getDocumentElement();
373        NodeList children = root.getChildNodes();
374        int len=children.getLength();
375        for(int i=0;i<len;i++) {
376            Node n=children.item(i);
377            if(n instanceof Element && n.getNodeName().equals("collection")) {
378                readCollection(config,(Element)n);
379            }
380        }
381    }
382
383    /**
384     * read in a single collection element
385     * @param config 
386     * @param el
387     * @throws SearchException
388     */
389    private final void readCollection(Config config, Element el) throws SearchException {
390        SearchCollectionPlus sc;
391        //try {
392            // Collection
393            DateTime last = DateCaster.toDateAdvanced(el.getAttribute("lastUpdate"),ThreadLocalPageContext.getTimeZone(config),null);
394            if(last==null)last=new DateTimeImpl();
395            DateTime cre = DateCaster.toDateAdvanced(el.getAttribute("created"),ThreadLocalPageContext.getTimeZone(config),null);
396            if(cre==null)cre=new DateTimeImpl();
397            ResourceProvider frp = ResourcesImpl.getFileResourceProvider();
398            sc =(SearchCollectionPlus) _readCollection(
399                    el.getAttribute("name"),
400                    frp.getResource(el.getAttribute("path")),
401                    el.getAttribute("language"),
402                    last,cre
403            );
404            collections.setEL(KeyImpl.init(sc.getName()),sc);
405            
406            // Indexes
407            NodeList children = el.getChildNodes();
408            int len=children.getLength();
409            for(int i=0;i<len;i++) {
410                Node n=children.item(i);
411                if(n instanceof Element && n.getNodeName().equals("index")) {
412                    readIndex(sc,(Element)n);
413                }
414            }
415        /*} 
416        catch (PageException e) {
417            throw new SearchException(e);
418        }*/
419    }
420
421    /**
422     * read in a single Index
423     * @param sc
424     * @param el
425     * @throws SearchException
426     * @throws PageException
427     */
428    protected void readIndex(SearchCollectionPlus sc, Element el) throws SearchException {
429            // Index
430            SearchIndex si=new SearchIndex(
431                    _attr(el,"id"),
432                    _attr(el,"title"),
433                    _attr(el,"key"),
434                    SearchIndex.toType(_attr(el,"type")),
435                    _attr(el,"query"),
436                    ListUtil.listToStringArray(_attr(el,"extensions"),','),
437                    _attr(el,"language"),
438                    _attr(el,"urlpath"),
439                    _attr(el,"categoryTree"),
440                    ListUtil.listToStringArray(_attr(el,"category"),','),
441                    _attr(el,"custom1"),
442                    _attr(el,"custom2"),
443                    _attr(el,"custom3"),
444                    _attr(el,"custom4"));
445           sc.addIndex(si);
446    }
447
448  private String _attr(Element el, String attr) {
449        return StringUtil.emptyIfNull(el.getAttribute(attr));
450    }
451
452      /**
453     * read in a existing collection
454     * @param name
455     * @param parh
456     * @param language
457     * @param count
458     * @param lastUpdate
459     * @param created 
460     * @return SearchCollection
461     * @throws SearchException
462     */
463    protected abstract SearchCollection _readCollection(String name, Resource parh, String language, DateTime lastUpdate, DateTime created) throws SearchException;
464
465        /**
466     * store loaded data to xml file
467         * @throws SearchException
468     */
469    protected final synchronized void store() throws SearchException {
470        //Collection.Key[] keys=collections.keys();
471        Iterator<Key> it = collections.keyIterator();
472        Key k;
473        while(it.hasNext()) {
474                k=it.next();
475            Element collEl = getCollectionElement(k.getString());
476            SearchCollection sc = getCollectionByName(k.getString());
477            setAttributes(collEl,sc);  
478        }
479
480        OutputFormat format = new OutputFormat(doc, null, true);
481                format.setLineSeparator("\r\n");
482                format.setLineWidth(72);
483                OutputStream os=null;
484                try {
485                    XMLSerializer serializer = new XMLSerializer(os=IOUtil.toBufferedOutputStream(searchFile.getOutputStream()), format);
486                        serializer.serialize(doc.getDocumentElement());
487                } catch (IOException e) {
488                    throw new SearchException(e);
489                }
490                finally {
491                        IOUtil.closeEL(os);
492                }
493    }
494
495    /**
496         * if no search xml exist create a empty one
497         * @param searchFile 
498         * @throws IOException
499         */
500        private final static void createSearchFile(Resource searchFile) throws IOException {
501        
502                searchFile.createFile(true);
503                InputStream in = new Info().getClass().getResourceAsStream("/resource/search/default.xml");
504                IOUtil.copy(in,searchFile,true);
505        
506        }
507    
508    @Override
509    public abstract String getDisplayName();
510
511        public Config getConfig() { 
512                return config;
513        }
514}