001/**
002 * Copyright (c) 2014, the Railo Company Ltd.
003 * Copyright (c) 2015, Lucee Assosication Switzerland
004 *
005 * This library is free software; you can redistribute it and/or
006 * modify it under the terms of the GNU Lesser General Public
007 * License as published by the Free Software Foundation; either 
008 * version 2.1 of the License, or (at your option) any later version.
009 * 
010 * This library is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
013 * Lesser General Public License for more details.
014 * 
015 * You should have received a copy of the GNU Lesser General Public 
016 * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
017 * 
018 **/
019package lucee.runtime.tag;
020
021import java.io.ByteArrayInputStream;
022import java.io.EOFException;
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import java.io.InputStream;
026import java.io.PrintStream;
027import java.io.UnsupportedEncodingException;
028import java.net.MalformedURLException;
029import java.net.SocketTimeoutException;
030import java.net.URL;
031import java.nio.charset.Charset;
032import java.util.ArrayList;
033import java.util.Iterator;
034import java.util.List;
035import java.util.concurrent.TimeUnit;
036import java.util.zip.GZIPInputStream;
037
038import javax.net.ssl.SSLContext;
039
040import lucee.commons.io.CharsetUtil;
041import lucee.commons.io.IOUtil;
042import lucee.commons.io.SystemUtil;
043import lucee.commons.io.res.Resource;
044import lucee.commons.io.res.util.ResourceUtil;
045import lucee.commons.lang.ExceptionUtil;
046import lucee.commons.lang.StringUtil;
047import lucee.commons.lang.mimetype.ContentType;
048import lucee.commons.net.HTTPUtil;
049import lucee.commons.net.http.httpclient4.CachingGZIPInputStream;
050import lucee.commons.net.http.httpclient4.HTTPEngineImpl;
051import lucee.commons.net.http.httpclient4.HTTPPatchFactory;
052import lucee.commons.net.http.httpclient4.HTTPResponse4Impl;
053import lucee.commons.net.http.httpclient4.ResourceBody;
054import lucee.runtime.PageContext;
055import lucee.runtime.PageContextImpl;
056import lucee.runtime.config.ConfigWeb;
057import lucee.runtime.engine.ThreadLocalPageContext;
058import lucee.runtime.exp.ApplicationException;
059import lucee.runtime.exp.ExpressionException;
060import lucee.runtime.exp.HTTPException;
061import lucee.runtime.exp.NativeException;
062import lucee.runtime.exp.PageException;
063import lucee.runtime.exp.RequestTimeoutException;
064import lucee.runtime.ext.tag.BodyTagImpl;
065import lucee.runtime.net.http.MultiPartResponseUtils;
066import lucee.runtime.net.http.ReqRspUtil;
067import lucee.runtime.net.http.sni.DefaultHostnameVerifierImpl;
068import lucee.runtime.net.http.sni.SSLConnectionSocketFactoryImpl;
069import lucee.runtime.net.http.sni.DefaultHttpClientConnectionOperatorImpl;
070import lucee.runtime.net.proxy.ProxyData;
071import lucee.runtime.net.proxy.ProxyDataImpl;
072import lucee.runtime.op.Caster;
073import lucee.runtime.text.csv.CSVParser;
074import lucee.runtime.type.Array;
075import lucee.runtime.type.ArrayImpl;
076import lucee.runtime.type.Collection.Key;
077import lucee.runtime.type.KeyImpl;
078import lucee.runtime.type.Query;
079import lucee.runtime.type.QueryImpl;
080import lucee.runtime.type.Struct;
081import lucee.runtime.type.StructImpl;
082import lucee.runtime.type.dt.DateTime;
083import lucee.runtime.type.dt.TimeSpan;
084import lucee.runtime.type.dt.TimeSpanImpl;
085import lucee.runtime.type.util.ArrayUtil;
086import lucee.runtime.type.util.KeyConstants;
087import lucee.runtime.type.util.ListUtil;
088import lucee.runtime.util.PageContextUtil;
089import lucee.runtime.util.URLResolver;
090
091import org.apache.http.HttpEntityEnclosingRequest;
092import org.apache.http.HttpHost;
093import org.apache.http.NameValuePair;
094import org.apache.http.client.methods.HttpDelete;
095import org.apache.http.client.methods.HttpGet;
096import org.apache.http.client.methods.HttpHead;
097import org.apache.http.client.methods.HttpOptions;
098import org.apache.http.client.methods.HttpPost;
099import org.apache.http.client.methods.HttpPut;
100import org.apache.http.client.methods.HttpRequestBase;
101import org.apache.http.client.methods.HttpTrace;
102import org.apache.http.config.Lookup;
103import org.apache.http.config.RegistryBuilder;
104import org.apache.http.config.SocketConfig;
105import org.apache.http.conn.socket.ConnectionSocketFactory;
106import org.apache.http.conn.socket.PlainConnectionSocketFactory;
107import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
108import org.apache.http.entity.StringEntity;
109import org.apache.http.entity.mime.FormBodyPart;
110import org.apache.http.entity.mime.HttpMultipartMode;
111import org.apache.http.entity.mime.MultipartEntity;
112import org.apache.http.entity.mime.content.ContentBody;
113import org.apache.http.entity.mime.content.StringBody;
114import org.apache.http.impl.client.BasicCookieStore;
115import org.apache.http.impl.client.CloseableHttpClient;
116import org.apache.http.impl.client.DefaultRedirectStrategy;
117import org.apache.http.impl.client.HttpClientBuilder;
118import org.apache.http.impl.client.HttpClients;
119import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
120import org.apache.http.message.BasicNameValuePair;
121import org.apache.http.protocol.BasicHttpContext;
122import org.apache.http.protocol.HttpContext;
123import org.apache.http.ssl.SSLContexts;
124
125// MUST change behavor of mltiple headers now is a array, it das so?
126
127/**
128* Lets you execute HTTP POST and GET operations on files. Using cfhttp, you can execute standard 
129*   GET operations and create a query object from a text file. POST operations lets you upload MIME file 
130*   types to a server, or post cookie, formfield, URL, file, or CGI variables directly to a specified server.
131*
132*
133*
134* 
135**/
136public final class Http41 extends BodyTagImpl implements Http {
137
138        public static final String MULTIPART_RELATED = "multipart/related";
139        public static final String MULTIPART_FORM_DATA = "multipart/form-data";
140
141    /**
142     * Maximum redirect count (5)
143     */
144    public static final short MAX_REDIRECT=15;
145    
146    /**
147     * Constant value for HTTP Status Code "moved Permanently 301"
148     */
149    public static final int STATUS_REDIRECT_MOVED_PERMANENTLY=301;
150    /**
151     * Constant value for HTTP Status Code "Found 302"
152     */
153    public static final int STATUS_REDIRECT_FOUND=302;
154    /**
155     * Constant value for HTTP Status Code "see other 303"
156     */
157    public static final int STATUS_REDIRECT_SEE_OTHER=303;
158    
159
160    public static final int STATUS_REDIRECT_TEMPORARY_REDIRECT = 307;
161
162
163        
164    
165    
166
167        private static final short METHOD_GET=0;
168        private static final short METHOD_POST=1;
169        private static final short METHOD_HEAD=2;
170        private static final short METHOD_PUT=3;
171        private static final short METHOD_DELETE=4;
172        private static final short METHOD_OPTIONS=5;
173        private static final short METHOD_TRACE=6;
174        private static final short METHOD_PATCH=7;
175        
176        private static final String NO_MIMETYPE="Unable to determine MIME type of file.";
177        
178        private static final short GET_AS_BINARY_NO=0;
179        private static final short GET_AS_BINARY_YES=1;
180        private static final short GET_AS_BINARY_AUTO=2;
181
182        private static final Key STATUSCODE = KeyConstants._statuscode;
183        private static final Key CHARSET = KeyConstants._charset;
184        
185        private static final Key ERROR_DETAIL = KeyImpl.intern("errordetail");
186        private static final Key STATUS_CODE = KeyImpl.intern("status_code");
187        private static final Key STATUS_TEXT = KeyImpl.intern("status_text");
188        private static final Key HTTP_VERSION = KeyImpl.intern("http_version");
189        
190
191        private static final Key FILE_CONTENT = KeyImpl.intern("filecontent");
192        private static final Key EXPLANATION = KeyImpl.intern("explanation");
193        private static final Key RESPONSEHEADER = KeyImpl.intern("responseheader");
194        private static final Key SET_COOKIE = KeyImpl.intern("set-cookie");
195        
196        private static final short AUTH_TYPE_BASIC = 0;
197        private static final short AUTH_TYPE_NTLM = 1;
198
199        
200        
201        
202        static {
203            //Protocol myhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
204            //Protocol.registerProtocol("https", new Protocol("https", new EasySSLProtocolSocketFactory(), 443));
205        }
206        
207
208    private ArrayList<HttpParamBean> params=new ArrayList<HttpParamBean>();
209        
210        
211        /** When required by a server, a valid password. */
212        private String password;
213
214        /** Required for creating a query. Options are a tab or comma. Default is a comma. */
215        private char delimiter=',';
216
217        /** Yes or No. Default is No. For GET and POST operations, if Yes, page reference returned into the 
218        **      fileContent internal variable has its internal URLs fully resolved, including port number, so that 
219        **      links remain intact. */
220        private boolean resolveurl;
221
222        /** A value, in seconds. When a URL timeout is specified in the browser */
223        private TimeSpan timeout=null;
224
225        /** Host name or IP address of a proxy server. */
226        private String proxyserver;
227
228        /** The filename to be used for the file that is accessed. For GET operations, defaults to the name 
229        **      pecified in url. Enter path information in the path attribute. */
230        private String strFile;
231
232        /** The path to the directory in which a file is to be stored. If a path is not specified in a POST 
233        **      or GET operation, a variable is created (cfhttp.fileContent) that you can use to display the results 
234        **      of the POST operation in a cfoutput. */
235        private String strPath;
236
237        /** Boolean indicating whether to throw an exception that can be caught by using the cftry and 
238        **      cfcatch tags. The default is NO. */
239        private boolean throwonerror;
240
241        /** set the charset for the call. */
242        private String charset=null;
243
244        /** The port number on the proxy server from which the object is requested. Default is 80. When 
245        **      used with resolveURL, the URLs of retrieved documents that specify a port number are automatically 
246        **      resolved to preserve links in the retrieved document. */
247        private int proxyport=80;
248
249        /** Specifies the column names for a query when creating a query as a result of a cfhttp GET. */
250        private String[] columns;
251
252        /** The port number on the server from which the object is requested. Default is 80. When used with 
253        **      resolveURL, the URLs of retrieved documents that specify a port number are automatically resolved to 
254        **      preserve links in the retrieved document. If a port number is specified in the url attribute, the port
255        **      value overrides the value of the port attribute. */
256        private int port=-1;
257
258        /** User agent request header. */
259        private String useragent="Lucee (CFML Engine)";
260
261        /** Required for creating a query. Indicates the start and finish of a column. Should be 
262        **      appropriately escaped when embedded in a column. For example, if the qualifier is a double quotation 
263        **      mark, it should be escaped as """". If there is no text qualifier in the file, specify it as " ". 
264        **      Default is the double quotation mark ("). */
265        private char textqualifier='"';
266
267        /** When required by a server, a valid username. */
268        private String username;
269
270        /** Full URL of the host name or IP address of the server on which the file resides. The URL must be
271        **      an absolute URL, including the protocol (http or https) and hostname. It may optionally contain a port
272        **      number. Port numbers specified in the url attribute override the port attribute. */
273        private String url;
274
275        /** Boolean indicating whether to redirect execution or stop execution.*/
276        private boolean redirect=true;
277
278
279        /** The name to assign to a query if the a query is constructed from a file. */
280        private String name;
281
282        /** GET or POST. Use GET to download a text or binary file or to create a query from the contents 
283        **      of a text file. Use POST to send information to a server page or a CGI program for processing. POST 
284        **      requires the use of a cfhttpparam tag. */
285        private short method=METHOD_GET;
286
287        //private boolean hasBody=false;
288        
289        private boolean firstrowasheaders=true;
290
291        private String proxyuser=null;
292        private String proxypassword="";
293        private boolean multiPart=false;
294        private String multiPartType=MULTIPART_FORM_DATA;
295        
296        private short getAsBinary=GET_AS_BINARY_NO;
297    private String result="cfhttp";
298    
299    private boolean addtoken=false;
300    
301    private short authType=AUTH_TYPE_BASIC;
302    private String workStation=null;
303    private String domain=null;
304        private boolean preauth=true;
305        private boolean encoded=true; 
306        
307        private boolean compression=true;
308
309        
310        public Http41() {
311                // Test if library is ok
312                Class<CloseableHttpClient> clazz = CloseableHttpClient.class;
313        }
314
315        @Override
316        public void release()   {
317                super.release();
318            params.clear();
319                password=null;
320                delimiter=',';
321                resolveurl=false;
322                timeout=null;
323                proxyserver=null;
324                proxyport=80;
325                proxyuser=null;
326                proxypassword="";
327                strFile=null;
328                throwonerror=false;
329                charset=null;
330                columns=null;
331                port=-1;
332                useragent="Lucee (CFML Engine)";
333                textqualifier='"';
334                username=null;
335                url=null;
336                redirect=true;
337                strPath=null;
338                name=null;
339                method=METHOD_GET;
340                //hasBody=false;
341                firstrowasheaders=true;
342                
343                getAsBinary=GET_AS_BINARY_NO;
344                multiPart=false;
345                multiPartType=MULTIPART_FORM_DATA;
346        result="cfhttp";
347        addtoken=false;
348        
349        authType=AUTH_TYPE_BASIC;
350        workStation=null;
351        domain=null;
352        preauth=true; 
353        encoded=true;
354        compression=true;
355        }
356        
357        /**
358         * @param firstrowasheaders
359         */
360        public void setFirstrowasheaders(boolean firstrowasheaders)     {
361                this.firstrowasheaders=firstrowasheaders;
362        }
363        
364
365        public void setEncodeurl(boolean encoded)       {
366                this.encoded=encoded;
367        }
368
369
370        /** set the value password
371        *  When required by a server, a valid password.
372        * @param password value to set
373        **/
374        public void setPassword(String password)        {
375                this.password=password;
376        }
377        /** set the value password
378        *  When required by a proxy server, a valid password.
379        * @param proxypassword value to set
380        **/
381        public void setProxypassword(String proxypassword)      {
382                this.proxypassword=proxypassword;
383        }
384
385        /** set the value delimiter
386        *  Required for creating a query. Options are a tab or comma. Default is a comma.
387        * @param delimiter value to set
388        **/
389        public void setDelimiter(String delimiter)      {
390                this.delimiter=delimiter.length()==0?',':delimiter.charAt(0);
391        }
392
393        /** set the value resolveurl
394        *  Yes or No. Default is No. For GET and POST operations, if Yes, page reference returned into the 
395        *       fileContent internal variable has its internal URLs fully resolved, including port number, so that 
396        *       links remain intact.
397        * @param resolveurl value to set
398        **/
399        public void setResolveurl(boolean resolveurl)   {
400                this.resolveurl=resolveurl;
401        }
402        
403        public void setPreauth(boolean preauth) {
404                this.preauth=preauth;
405        }
406        
407        
408
409        /** set the value timeout
410        * @param timeout value to set
411         * @throws ExpressionException 
412        **/
413        public void setTimeout(Object timeout) throws PageException     {
414                if(timeout instanceof TimeSpan)
415                        this.timeout=(TimeSpan) timeout;
416                // seconds
417                else {
418                        int i = Caster.toIntValue(timeout);
419                        if(i<0)
420                                throw new ApplicationException("invalid value ["+i+"] for attribute timeout, value must be a positive integer greater or equal than 0");
421                        
422                        this.timeout=new TimeSpanImpl(0, 0, 0, i);
423                }
424        }
425
426        /** set the value proxyserver
427        *  Host name or IP address of a proxy server.
428        * @param proxyserver value to set
429        **/
430        public void setProxyserver(String proxyserver)  {
431                this.proxyserver=proxyserver;
432        }
433        
434        /** set the value proxyport
435        *  The port number on the proxy server from which the object is requested. Default is 80. When 
436        *       used with resolveURL, the URLs of retrieved documents that specify a port number are automatically 
437        *       resolved to preserve links in the retrieved document.
438        * @param proxyport value to set
439        **/
440        public void setProxyport(double proxyport)      {
441                this.proxyport=(int)proxyport;
442        }
443
444        /** set the value file
445        *  The filename to be used for the file that is accessed. For GET operations, defaults to the name 
446        *       pecified in url. Enter path information in the path attribute.
447        * @param file value to set
448        **/
449        public void setFile(String file)        {
450                this.strFile=file;
451        }
452
453        /** set the value throwonerror
454        *  Boolean indicating whether to throw an exception that can be caught by using the cftry and 
455        *       cfcatch tags. The default is NO.
456        * @param throwonerror value to set
457        **/
458        public void setThrowonerror(boolean throwonerror)       {
459                this.throwonerror=throwonerror;
460        }
461
462        /** set the value charset
463        *  set the charset for the call.
464        * @param charset value to set
465        **/
466        public void setCharset(String charset)  {
467                this.charset=charset;
468        }
469
470        /** set the value columns
471        * @param columns value to set
472         * @throws PageException
473        **/
474        public void setColumns(String columns) throws PageException     {
475                this.columns=ListUtil.toStringArray(ListUtil.listToArrayRemoveEmpty(columns,","));
476        }
477
478        /** set the value port
479        *  The port number on the server from which the object is requested. Default is 80. When used with 
480        *       resolveURL, the URLs of retrieved documents that specify a port number are automatically resolved to 
481        *       preserve links in the retrieved document. If a port number is specified in the url attribute, the port
482        *       value overrides the value of the port attribute.
483        * @param port value to set
484        **/
485        public void setPort(double port)        {
486                this.port=(int) port;
487        }
488
489        /** set the value useragent
490        *  User agent request header.
491        * @param useragent value to set
492        **/
493        public void setUseragent(String useragent)      {
494                this.useragent=useragent;
495        }
496
497        /** set the value textqualifier
498        *  Required for creating a query. Indicates the start and finish of a column. Should be 
499        *       appropriately escaped when embedded in a column. For example, if the qualifier is a double quotation 
500        *       mark, it should be escaped as """". If there is no text qualifier in the file, specify it as " ". 
501        *       Default is the double quotation mark (").
502        * @param textqualifier value to set
503        **/
504        public void setTextqualifier(String textqualifier)      {
505                this.textqualifier=textqualifier.length()==0?'"':textqualifier.charAt(0);
506        }
507
508        /** set the value username
509        *  When required by a proxy server, a valid username.
510        * @param proxyuser value to set
511        **/
512        public void setProxyuser(String proxyuser)      {
513                this.proxyuser=proxyuser;
514        }
515
516        /** set the value username
517        *  When required by a server, a valid username.
518        * @param username value to set
519        **/
520        public void setUsername(String username)        {
521                this.username=username;
522        }
523
524        /** set the value url
525        *  Full URL of the host name or IP address of the server on which the file resides. The URL must be
526        *       an absolute URL, including the protocol (http or https) and hostname. It may optionally contain a port
527        *       number. Port numbers specified in the url attribute override the port attribute.
528        * @param url value to set
529        **/
530        public void setUrl(String url)  {
531                this.url=url;
532        }
533
534        /** set the value redirect
535        * @param redirect value to set
536        **/
537        public void setRedirect(boolean redirect)       {
538                this.redirect=redirect;
539        }
540
541        /** set the value path
542        *  The path to the directory in which a file is to be stored. If a path is not specified in a POST 
543        *       or GET operation, a variable is created (cfhttp.fileContent) that you can use to display the results 
544        *       of the POST operation in a cfoutput.
545        * @param path value to set
546        **/
547        public void setPath(String path)        {
548                this.strPath=path;
549        }
550
551        /** set the value name
552        *  The name to assign to a query if the a query is constructed from a file.
553        * @param name value to set
554        **/
555        public void setName(String name)        {
556                this.name=name;
557        }
558        
559        public void setAuthtype(String strAuthType) throws ExpressionException{
560                if(StringUtil.isEmpty(strAuthType,true)) return;
561                strAuthType=strAuthType.trim();
562                if("basic".equalsIgnoreCase(strAuthType)) authType=AUTH_TYPE_BASIC;
563                else if("ntlm".equalsIgnoreCase(strAuthType)) authType=AUTH_TYPE_NTLM;
564                else
565                        throw new ExpressionException("invalid value ["+strAuthType+"] for attribute authType, value must be one of the following [basic,ntlm]");
566        }
567
568        public void setWorkstation(String workStation)  {
569                this.workStation=workStation;
570        }
571
572        public void setDomain(String domain)    {
573                this.domain=domain;
574        }
575
576        /** set the value method
577        *  GET or POST. Use GET to download a text or binary file or to create a query from the contents 
578        *       of a text file. Use POST to send information to a server page or a CGI program for processing. POST 
579        *       requires the use of a cfhttpparam tag.
580        * @param method value to set
581         * @throws ApplicationException
582        **/
583        public void setMethod(String method) throws ApplicationException        {
584            method=method.toLowerCase().trim();
585            if(method.equals("post")) this.method=METHOD_POST;
586            else if(method.equals("get")) this.method=METHOD_GET;
587            else if(method.equals("head")) this.method=METHOD_HEAD;
588            else if(method.equals("delete")) this.method=METHOD_DELETE;
589            else if(method.equals("put")) this.method=METHOD_PUT;
590            else if(method.equals("trace")) this.method=METHOD_TRACE;
591            else if(method.equals("options")) this.method=METHOD_OPTIONS;
592            else if(method.equals("patch")) this.method=METHOD_PATCH;
593            else throw new ApplicationException("invalid method type ["+(method.toUpperCase())+"], valid types are POST,GET,HEAD,DELETE,PUT,TRACE,OPTIONS,PATCH");
594        }
595        
596        public void setCompression(String strCompression) throws ApplicationException {
597                if(StringUtil.isEmpty(strCompression,true)) return;
598                Boolean b = Caster.toBoolean(strCompression,null);
599                
600                if(b!=null) compression=b.booleanValue();
601                else if(strCompression.trim().equalsIgnoreCase("none")) compression=false;
602            else throw new ApplicationException("invalid value for attribute compression ["+strCompression+"], valid values are: true,false or none");
603                
604        }
605
606
607        @Override
608        public int doStartTag() {
609                if(addtoken) {
610                        setParam("cookie","cfid",pageContext.getCFID());
611                        setParam("cookie","cftoken",pageContext.getCFToken());
612                        String jsessionid = pageContext.getJSessionId();
613                        if(jsessionid!=null)setParam("cookie","jsessionid",jsessionid);
614                }
615                
616                return EVAL_BODY_INCLUDE;
617        }
618
619        private void setParam(String type, String name, String value) {
620                HttpParamBean hpb = new HttpParamBean();
621                hpb.setType(type);
622                hpb.setName(name);
623                hpb.setValue(value);
624                setParam(hpb);
625        }
626
627        @Override
628        public int doEndTag() throws PageException {
629            Struct cfhttp=new StructImpl();
630                cfhttp.setEL(ERROR_DETAIL,"");
631                pageContext.setVariable(result,cfhttp);
632
633                // because commons 
634                PrintStream out = System.out;
635        try {
636                //System.setOut(new PrintStream(DevNullOutputStream.DEV_NULL_OUTPUT_STREAM));
637             _doEndTag(cfhttp);
638             return EVAL_PAGE;
639        } 
640        catch (IOException e) {
641            throw Caster.toPageException(e);
642        }
643        finally {
644                System.setOut(out);
645        }
646        }
647
648        
649        
650        private void _doEndTag(Struct cfhttp) throws PageException, IOException {
651                HttpClientBuilder builder = HttpClients.custom();
652                //SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext) {
653                ssl(builder);
654                
655        // redirect
656        if(redirect)  builder.setRedirectStrategy(new DefaultRedirectStrategy());
657        else builder.disableRedirectHandling();
658        
659        // cookies
660        BasicCookieStore cookieStore = new BasicCookieStore();
661        builder.setDefaultCookieStore(cookieStore);
662
663        ConfigWeb cw = pageContext.getConfig();
664        HttpRequestBase req=null;
665        HttpContext httpContext=null;
666        //HttpRequestBase req = init(pageContext.getConfig(),this,client,params,url,port);
667        {
668                if(StringUtil.isEmpty(charset,true)) charset=((PageContextImpl)pageContext).getWebCharset().name();
669                else charset=charset.trim();
670                
671                
672        // check if has fileUploads     
673                boolean doUploadFile=false;
674                for(int i=0;i<this.params.size();i++) {
675                        if((this.params.get(i)).getType().equalsIgnoreCase("file")) {
676                                doUploadFile=true;
677                                break;
678                        }
679                }       
680        
681        // parse url (also query string)
682                int len=this.params.size();
683                StringBuilder sbQS=new StringBuilder();
684                for(int i=0;i<len;i++) {
685                        HttpParamBean param=this.params.get(i);
686                        String type=param.getType();
687                // URL
688                        if(type.equals("url")) {
689                                if(sbQS.length()>0)sbQS.append('&');
690                                sbQS.append(param.getEncoded()?HttpImpl.urlenc(param.getName(),charset):param.getName());
691                                sbQS.append('=');
692                                sbQS.append(param.getEncoded()?HttpImpl.urlenc(param.getValueAsString(), charset):param.getValueAsString());
693                        }
694                }
695                String host=null;
696                HttpHost httpHost;
697                try {
698                        URL _url = HTTPUtil.toURL(url,port,encoded);
699                        httpHost = new HttpHost(_url.getHost(),_url.getPort());
700                        host=_url.getHost();
701                        url=_url.toExternalForm();
702                        if(sbQS.length()>0){
703                                // no existing QS
704                                if(StringUtil.isEmpty(_url.getQuery())) {
705                                        url+="?"+sbQS;
706                                }
707                                else {
708                                        url+="&"+sbQS;
709                                }
710                        }
711                } 
712                catch (MalformedURLException mue) {
713                        throw Caster.toPageException(mue);
714                }
715                
716        // select best matching method (get,post, post multpart (file))
717
718                boolean isBinary = false;
719                boolean doMultiPart=doUploadFile || this.multiPart;
720                HttpEntityEnclosingRequest post=null;
721                HttpEntityEnclosingRequest eem=null;
722                
723                
724                if(this.method==METHOD_GET) {
725                        req=new HttpGet(url);
726                }
727                else if(this.method==METHOD_HEAD) {
728                    req=new HttpHead(url);
729                }
730                else if(this.method==METHOD_DELETE) {
731                        isBinary=true;
732                    req=new HttpDelete(url);
733                }
734                else if(this.method==METHOD_PUT) {
735                        isBinary=true;
736                        HttpPut put = new HttpPut(url);
737                        post=put;
738                    req=put;
739                    eem=put;
740                    
741                }
742                else if(this.method==METHOD_TRACE) {
743                        isBinary=true;
744                    req=new HttpTrace(url);
745                }
746                else if(this.method==METHOD_OPTIONS) {
747                        isBinary=true;
748                    req=new HttpOptions(url);
749                }
750                else if(this.method==METHOD_PATCH) {
751                        isBinary=true;
752                        eem = HTTPPatchFactory.getHTTPPatch(url);
753                    req=(HttpRequestBase) eem;
754                }
755                else {
756                        isBinary=true;
757                        post=new HttpPost(url);
758                        req=(HttpPost)post;
759                        eem=post;
760                }
761                
762                boolean hasForm=false;
763                boolean hasBody=false;
764                boolean hasContentType=false;
765        // Set http params
766                ArrayList<FormBodyPart> parts=new ArrayList<FormBodyPart>();
767                
768                StringBuilder acceptEncoding=new StringBuilder();
769                java.util.List<NameValuePair> postParam = post!=null?new ArrayList <NameValuePair>():null;
770            
771                for(int i=0;i<len;i++) {
772                        HttpParamBean param=this.params.get(i);
773                        String type=param.getType();
774                        
775                // URL
776                        if(type.equals("url")) {
777                                //listQS.add(new BasicNameValuePair(translateEncoding(param.getName(), http.charset),translateEncoding(param.getValueAsString(), http.charset)));
778                        }
779                // Form
780                        else if(type.equals("formfield") || type.equals("form")) {
781                                hasForm=true;
782                                if(this.method==METHOD_GET) throw new ApplicationException("httpparam with type formfield can only be used when the method attribute of the parent http tag is set to post");
783                                if(post!=null){
784                                        if(doMultiPart) {
785                                                parts.add(
786                                                        new FormBodyPart(
787                                                                param.getName(),
788                                                                new StringBody(
789                                                                                param.getValueAsString(),
790                                                                                CharsetUtil.toCharset(charset)
791                                                                )
792                                                        )
793                                                );
794                                        }
795                                        else {
796                                                postParam.add(new BasicNameValuePair(param.getName(),param.getValueAsString()));
797                                        }
798                                }
799                                //else if(multi!=null)multi.addParameter(param.getName(),param.getValueAsString());
800                        }
801                // CGI
802                        else if(type.equals("cgi")) {
803                                if(param.getEncoded())
804                                        req.addHeader(
805                                                        HttpImpl.urlenc(param.getName(),charset),
806                                                        HttpImpl.urlenc(param.getValueAsString(),charset));
807                    else
808                        req.addHeader(param.getName(),param.getValueAsString());
809                        }
810            // Header
811                else if(type.startsWith("head")) {
812                        if(param.getName().equalsIgnoreCase("content-type")) hasContentType=true;
813                        
814                        if(param.getName().equalsIgnoreCase("Content-Length")) {}
815                        else if(param.getName().equalsIgnoreCase("Accept-Encoding")) {
816                                acceptEncoding.append(HttpImpl.headerValue(param.getValueAsString()));
817                                acceptEncoding.append(", ");
818                        }
819                        else req.addHeader(param.getName(),HttpImpl.headerValue(param.getValueAsString()));
820                }
821                // Cookie
822                        else if(type.equals("cookie")) {
823                                HTTPEngineImpl.addCookie(cookieStore,host,param.getName(),param.getValueAsString(),"/",charset);
824                        }
825                // File
826                        else if(type.equals("file")) {
827                                hasForm=true;
828                                if(this.method==METHOD_GET) throw new ApplicationException("httpparam type file can't only be used, when method of the tag http equal post");
829                                String strCT = HttpImpl.getContentType(param);
830                                ContentType ct = HTTPUtil.toContentType(strCT,null);
831                                
832                                String mt="text/xml";
833                                if(ct!=null && !StringUtil.isEmpty(ct.getMimeType(),true)) mt=ct.getMimeType();
834                                
835                                String cs=charset;
836                                if(ct!=null && !StringUtil.isEmpty(ct.getCharset(),true)) cs=ct.getCharset();
837                                
838                                
839                                if(doMultiPart) {
840                                        try {
841                                                Resource res = param.getFile();
842                                                parts.add(new FormBodyPart(
843                                                                param.getName(), 
844                                                                new ResourceBody(res, mt, res.getName(), cs)
845                                                ));
846                                                //parts.add(new ResourcePart(param.getName(),new ResourcePartSource(param.getFile()),getContentType(param),_charset));
847                                        } 
848                                        catch (FileNotFoundException e) {
849                                                throw new ApplicationException("can't upload file, path is invalid",e.getMessage());
850                                        }
851                                }
852                        }
853                // XML
854                        else if(type.equals("xml")) {
855                                ContentType ct = HTTPUtil.toContentType(param.getMimeType(),null);
856                                
857                                String mt="text/xml";
858                                if(ct!=null && !StringUtil.isEmpty(ct.getMimeType(),true)) mt=ct.getMimeType();
859                                
860                                String cs=charset;
861                                if(ct!=null && !StringUtil.isEmpty(ct.getCharset(),true)) cs=ct.getCharset();
862                                
863                                hasBody=true;
864                                hasContentType=true;
865                                req.addHeader("Content-type", mt+"; charset="+cs);
866                            if(eem==null)throw new ApplicationException("type xml is only supported for type post and put");
867                            HTTPEngineImpl.setBody(eem, param.getValueAsString(),mt,cs);
868                        }
869                // Body
870                        else if(type.equals("body")) {
871                                ContentType ct = HTTPUtil.toContentType(param.getMimeType(),null);
872                                
873                                String mt=null;
874                                if(ct!=null && !StringUtil.isEmpty(ct.getMimeType(),true)) mt=ct.getMimeType();
875                                
876                                String cs=charset;
877                                if(ct!=null && !StringUtil.isEmpty(ct.getCharset(),true)) cs=ct.getCharset();
878                                
879                                
880                                hasBody=true;
881                                if(eem==null)throw new ApplicationException("type body is only supported for type post and put");
882                                HTTPEngineImpl.setBody(eem, param.getValue(),mt,cs);
883                                
884                        }
885                else {
886                    throw new ApplicationException("invalid type ["+type+"]");
887                }
888                    
889                }
890                
891                // post params
892                if(postParam!=null && postParam.size()>0)
893                        post.setEntity(new org.apache.http.client.entity.UrlEncodedFormEntity(postParam,charset));
894                
895                if(compression){
896                        acceptEncoding.append("gzip");
897                }
898                else {
899                        acceptEncoding.append("deflate;q=0");
900                        req.setHeader("TE", "deflate;q=0");
901                }
902                        req.setHeader("Accept-Encoding",acceptEncoding.toString());
903                
904                
905                
906                // multipart
907                if(doMultiPart && eem!=null) {
908                        hasContentType=true;
909                        boolean doIt=true;
910                        if(!this.multiPart && parts.size()==1){
911                                ContentBody body = parts.get(0).getBody();
912                                if(body instanceof StringBody){
913                                        StringBody sb=(StringBody)body;
914                                        try {
915                                                org.apache.http.entity.ContentType ct=org.apache.http.entity.ContentType.create(sb.getMimeType(),sb.getCharset());
916                                                String str = IOUtil.toString(sb.getReader());
917                                                StringEntity entity = new StringEntity(str,ct);
918                                                eem.setEntity(entity);
919                                                
920                                        } catch (IOException e) {
921                                                throw Caster.toPageException(e);
922                                        }
923                                        doIt=false;
924                                }
925                        }
926                        if(doIt) {
927                                MultipartEntity mpe = new MultipartEntity(HttpMultipartMode.STRICT);
928                                Iterator<FormBodyPart> it = parts.iterator();
929                                while(it.hasNext()) {
930                                        FormBodyPart part = it.next();
931                                        mpe.addPart(part.getName(),part.getBody());
932                                }
933                                eem.setEntity(mpe);
934                        }
935                                //eem.setRequestEntity(new MultipartRequestEntityFlex(parts.toArray(new Part[parts.size()]), eem.getParams(),http.multiPartType));
936                }
937                
938                
939                
940                if(hasBody && hasForm)
941                        throw new ApplicationException("mixing httpparam  type file/formfield and body/XML is not allowed");
942        
943                if(!hasContentType) {
944                        if(isBinary) {
945                                if(hasBody) req.addHeader("Content-type", "application/octet-stream");
946                                else req.addHeader("Content-type", "application/x-www-form-urlencoded; charset="+charset);
947                        }
948                        else {
949                                if(hasBody)
950                                        req.addHeader("Content-type", "text/html; charset="+charset ); 
951                        }
952                }
953                
954                
955                // set User Agent
956                        if(!HttpImpl.hasHeaderIgnoreCase(req,"User-Agent"))
957                                req.setHeader("User-Agent",this.useragent);
958                
959                        setTimeout(builder,checkRemainingTimeout());
960
961        // set Username and Password
962                if(this.username!=null) {
963                        if(this.password==null)this.password="";
964                        if(AUTH_TYPE_NTLM==this.authType) {
965                                if(StringUtil.isEmpty(this.workStation,true))
966                        throw new ApplicationException("attribute workstation is required when authentication type is [NTLM]");
967                                if(StringUtil.isEmpty(this.domain,true))
968                        throw new ApplicationException("attribute domain is required when authentication type is [NTLM]");
969                                HTTPEngineImpl.setNTCredentials(builder, this.username, this.password, this.workStation,this.domain);
970                        }
971                        else httpContext=HTTPEngineImpl.setCredentials(builder, httpHost, this.username, this.password,preauth);
972                }
973        
974        // set Proxy
975                ProxyData proxy=null;
976                if(!StringUtil.isEmpty(this.proxyserver)) {
977                        proxy=ProxyDataImpl.getInstance(this.proxyserver, this.proxyport, this.proxyuser, this.proxypassword) ;
978                }
979                if(pageContext.getConfig().isProxyEnableFor(host)) { 
980                        proxy=pageContext.getConfig().getProxyData();
981                }
982                HTTPEngineImpl.setProxy(builder, req, proxy);
983                
984        }
985        
986        
987        
988        
989        CloseableHttpClient client=null;
990        try {
991        if(httpContext==null)httpContext = new BasicHttpContext();
992                
993/////////////////////////////////////////// EXECUTE /////////////////////////////////////////////////
994        client = builder.build();
995        Executor41 e = new Executor41(pageContext,this,client,httpContext,req,redirect);
996                HTTPResponse4Impl rsp=null;
997                if(timeout==null || timeout.getMillis()<=0) {// never happens
998                        try{
999                                rsp = e.execute(httpContext);
1000                        }
1001                        
1002                        catch(Throwable t){
1003                                ExceptionUtil.rethrowIfNecessary(t);
1004                                if(!throwonerror){
1005                                        if(t instanceof SocketTimeoutException)setRequestTimeout(cfhttp);
1006                                        else setUnknownHost(cfhttp, t);
1007                                        
1008                                        return;
1009                                }
1010                                throw toPageException(t,rsp);
1011                                
1012                        }
1013                }
1014                else {
1015                        e.start();
1016                        try {
1017                                synchronized(this){//print.err(timeout);
1018                                        this.wait(timeout.getMillis());
1019                                }
1020                        } 
1021                        catch (InterruptedException ie) {
1022                                throw Caster.toPageException(ie);
1023                        }
1024                        if(e.t!=null){
1025                                if(!throwonerror){
1026                                        setUnknownHost(cfhttp,e.t);
1027                                        return;
1028                                }
1029                                
1030                                throw toPageException(e.t,rsp); 
1031                        }
1032                        rsp=e.response;
1033                        if(!e.done){
1034                                req.abort();
1035                                if(throwonerror)
1036                                        throw new HTTPException("408 Request Time-out","a timeout occurred in tag http",408,"Time-out",rsp==null?null:rsp.getURL());
1037                                setRequestTimeout(cfhttp);      
1038                                return;
1039                                //throw new ApplicationException("timeout");    
1040                        }
1041                }
1042                
1043/////////////////////////////////////////// EXECUTE /////////////////////////////////////////////////
1044                Charset responseCharset=CharsetUtil.toCharset(rsp.getCharset());
1045        // Write Response Scope
1046                //String rawHeader=httpMethod.getStatusLine().toString();
1047                        String mimetype=null;
1048                        String contentEncoding=null;
1049                        
1050                // status code
1051                        cfhttp.set(STATUSCODE,((rsp.getStatusCode()+" "+rsp.getStatusText()).trim()));
1052                        cfhttp.set(STATUS_CODE,new Double(rsp.getStatusCode()));
1053                        cfhttp.set(STATUS_TEXT,(rsp.getStatusText()));
1054                        cfhttp.set(HTTP_VERSION,(rsp.getProtocolVersion()));
1055                        
1056                //responseHeader
1057                        lucee.commons.net.http.Header[] headers = rsp.getAllHeaders();
1058                        StringBuffer raw=new StringBuffer(rsp.getStatusLine()+" ");
1059                        Struct responseHeader = new StructImpl();
1060                        Struct cookie;
1061                        Array setCookie = new ArrayImpl();
1062                        Query cookies=new QueryImpl(new String[]{"name","value","path","domain","expires","secure","httpOnly"},0,"cookies");
1063                        
1064                for(int i=0;i<headers.length;i++) {
1065                        lucee.commons.net.http.Header header=headers[i];
1066                        //print.ln(header);
1067                        
1068                        raw.append(header.toString()+" ");
1069                        if(header.getName().equalsIgnoreCase("Set-Cookie")) {
1070                                setCookie.append(header.getValue());
1071                                parseCookie(cookies,header.getValue());
1072                        }
1073                        else {
1074                            //print.ln(header.getName()+"-"+header.getValue());
1075                                Object value=responseHeader.get(KeyImpl.getInstance(header.getName()),null);
1076                                if(value==null) responseHeader.set(KeyImpl.getInstance(header.getName()),header.getValue());
1077                                else {
1078                                    Array arr=null;
1079                                    if(value instanceof Array) {
1080                                        arr=(Array) value;
1081                                    }
1082                                    else {
1083                                        arr=new ArrayImpl();
1084                                        responseHeader.set(KeyImpl.getInstance(header.getName()),arr);
1085                                        arr.appendEL(value);
1086                                    }
1087                                    arr.appendEL(header.getValue());
1088                                }
1089                        }
1090                        
1091                        // Content-Type
1092                        if(header.getName().equalsIgnoreCase("Content-Type")) {
1093                                mimetype=header.getValue();
1094                            if(mimetype==null)mimetype=NO_MIMETYPE;
1095                        }
1096                        
1097                        // Content-Encoding
1098                        if(header.getName().equalsIgnoreCase("Content-Encoding")) {
1099                                contentEncoding=header.getValue();
1100                        }
1101                        
1102                }
1103                cfhttp.set(RESPONSEHEADER,responseHeader);
1104                cfhttp.set(KeyConstants._cookies,cookies);
1105                responseHeader.set(STATUS_CODE,new Double(rsp.getStatusCode()));
1106                responseHeader.set(EXPLANATION,(rsp.getStatusText()));
1107                if(setCookie.size()>0)responseHeader.set(SET_COOKIE,setCookie);
1108                
1109            // is text 
1110                boolean isText=
1111                        mimetype == null ||  
1112                        mimetype == NO_MIMETYPE || HTTPUtil.isTextMimeType(mimetype);
1113                        
1114                    // is multipart 
1115                boolean isMultipart= MultiPartResponseUtils.isMultipart(mimetype);        
1116               
1117                cfhttp.set(KeyConstants._text,Caster.toBoolean(isText));
1118                
1119            // mimetype charset
1120                //boolean responseProvideCharset=false;
1121                if(!StringUtil.isEmpty(mimetype,true)){
1122                        if(isText) {
1123                                String[] types=HTTPUtil.splitMimeTypeAndCharset(mimetype,null);
1124                                if(types[0]!=null)cfhttp.set(KeyConstants._mimetype,types[0]);
1125                                if(types[1]!=null)cfhttp.set(CHARSET,types[1]);
1126                        
1127                        }
1128                        else cfhttp.set(KeyConstants._mimetype,mimetype);
1129                }
1130                else cfhttp.set(KeyConstants._mimetype,NO_MIMETYPE);
1131
1132            // File
1133                Resource file=null;
1134                
1135                if(strFile!=null && strPath!=null) {
1136                    file=ResourceUtil.toResourceNotExisting(pageContext, strPath).getRealResource(strFile);
1137                }
1138                else if(strFile!=null) {
1139                    file=ResourceUtil.toResourceNotExisting(pageContext, strFile);
1140                }
1141                else if(strPath!=null) {
1142                    file=ResourceUtil.toResourceNotExisting(pageContext, strPath);
1143                    //Resource dir = file.getParentResource();
1144                    if(file.isDirectory()){
1145                        file=file.getRealResource(req.getURI().getPath());// TODO was getName() ->http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/URI.html#getName()
1146                    }
1147                    
1148                }
1149                if(file!=null)pageContext.getConfig().getSecurityManager().checkFileLocation(file);
1150                
1151                
1152                // filecontent
1153                InputStream is=null;
1154                    if(isText && getAsBinary!=GET_AS_BINARY_YES) {
1155                        String str;
1156                try {
1157                        
1158                        // read content
1159                        if(method!=METHOD_HEAD) {
1160                                is = rsp.getContentAsStream();
1161                            if(is!=null &&HttpImpl.isGzipEncoded(contentEncoding))
1162                                is = rsp.getStatusCode()!=200? new CachingGZIPInputStream(is):new GZIPInputStream(is);
1163                        }       
1164                    try {
1165                        try{
1166                        str = is==null?"":IOUtil.toString(is,responseCharset,checkRemainingTimeout().getMillis());
1167                        }
1168                        catch(EOFException eof){
1169                                if(is instanceof CachingGZIPInputStream) {
1170                                        str = IOUtil.toString(is=((CachingGZIPInputStream)is).getRawData(),responseCharset,checkRemainingTimeout().getMillis());
1171                                }
1172                                else throw eof;
1173                        }
1174                    }
1175                    catch (UnsupportedEncodingException uee) {
1176                        str = IOUtil.toString(is,(Charset)null,checkRemainingTimeout().getMillis());
1177                    }
1178                }
1179                catch (IOException ioe) {
1180                        throw Caster.toPageException(ioe);
1181                }
1182                finally {
1183                        IOUtil.closeEL(is);
1184                }
1185                    
1186                if(str==null)str="";
1187                        if(resolveurl){
1188                                //if(e.redirectURL!=null)url=e.redirectURL.toExternalForm();
1189                                str=new URLResolver().transform(str,e.response.getTargetURL(),false);
1190                        }
1191                        cfhttp.set(FILE_CONTENT,str);
1192                        try {
1193                                if(file!=null){
1194                                        IOUtil.write(file,str,((PageContextImpl)pageContext).getWebCharset(),false);
1195                    }
1196                } 
1197                        catch (IOException e1) {}
1198
1199                        if(name!=null) {
1200                    Query qry = CSVParser.toQuery( str, delimiter, textqualifier, columns, firstrowasheaders  );
1201                    pageContext.setVariable(name,qry);
1202                        }
1203                    }
1204                    // Binary
1205                    else {
1206                        byte[] barr=null;
1207                        if(HttpImpl.isGzipEncoded(contentEncoding)){
1208                                if(method!=METHOD_HEAD) {
1209                                        is=rsp.getContentAsStream();
1210                                        is = rsp.getStatusCode()!=200?new CachingGZIPInputStream(is) :new GZIPInputStream(is);
1211                                }
1212                                
1213                                try {
1214                                        try{
1215                                                barr = is==null?new byte[0]: IOUtil.toBytes(is);
1216                                        }
1217                                        catch(EOFException eof){
1218                                                if(is instanceof CachingGZIPInputStream)
1219                                                        barr = IOUtil.toBytes(((CachingGZIPInputStream)is).getRawData());
1220                                                else throw eof;
1221                                        }
1222                                        } 
1223                                catch (IOException t) {
1224                                        throw Caster.toPageException(t);
1225                                        }
1226                                        finally{
1227                                                IOUtil.closeEL(is);
1228                                        }
1229                        }
1230                        else {
1231                                try {
1232                                        if(method!=METHOD_HEAD) barr = rsp.getContentAsByteArray();
1233                                        else barr=new byte[0];
1234                                        } 
1235                                catch (IOException t) {
1236                                        throw Caster.toPageException(t);
1237                                        }
1238                        }
1239                        //IF Multipart response get file content and parse parts
1240                        if(barr!=null) {
1241                                    if(isMultipart) {
1242                                        cfhttp.set(FILE_CONTENT,MultiPartResponseUtils.getParts(barr,mimetype));
1243                                    } else {
1244                                        cfhttp.set(FILE_CONTENT,barr);
1245                                    }
1246                        }
1247                        else 
1248                                cfhttp.set(FILE_CONTENT,"");
1249                        
1250                        
1251                        if(file!=null) {
1252                                try {
1253                                        if(barr!=null)IOUtil.copy(new ByteArrayInputStream(barr),file,true);
1254                                } 
1255                                catch (IOException ioe) {
1256                                throw Caster.toPageException(ioe);
1257                                }
1258                        }   
1259                    }
1260                
1261            // header           
1262                cfhttp.set(KeyConstants._header,raw.toString());
1263                if(!HttpImpl.isStatusOK(rsp.getStatusCode())){
1264                        String msg=rsp.getStatusCode()+" "+rsp.getStatusText();
1265                    cfhttp.setEL(ERROR_DETAIL,msg);
1266                    if(throwonerror){
1267                        throw new HTTPException(msg,null,rsp.getStatusCode(),rsp.getStatusText(),rsp.getURL());
1268                    }
1269                }
1270                }
1271                finally {
1272                        if(client!=null)client.close();
1273                }
1274            
1275        }
1276        
1277        private void ssl(HttpClientBuilder builder) {
1278                SSLContext sslcontext = SSLContexts.createSystemDefault();
1279                
1280                final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactoryImpl(sslcontext,new DefaultHostnameVerifierImpl());
1281                builder.setSSLSocketFactory(sslsf);
1282                org.apache.http.config.Registry<ConnectionSocketFactory> reg = RegistryBuilder.<ConnectionSocketFactory>create()
1283        .register("http", PlainConnectionSocketFactory.getSocketFactory())
1284        .register("https", sslsf)
1285        .build();
1286                PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(
1287                                new DefaultHttpClientConnectionOperatorImpl(reg), null, -1, TimeUnit.MILLISECONDS); // TODO review -1 setting
1288                builder.setConnectionManager(cm);
1289        }
1290
1291        private TimeSpan checkRemainingTimeout() throws RequestTimeoutException {
1292                //timeout not defined
1293                if(this.timeout==null || ((int)timeout.getSeconds())<=0) { // not set
1294                        this.timeout=PageContextUtil.remainingTime(pageContext,true);
1295                }
1296                // timeout bigger than remaining time
1297                else {
1298                        TimeSpan remaining = PageContextUtil.remainingTime(pageContext,true);
1299                        if(timeout.getSeconds()>remaining.getSeconds())
1300                                timeout=remaining;
1301                }
1302                return timeout;
1303        }
1304
1305        public static void setTimeout(HttpClientBuilder builder, TimeSpan timeout) {
1306                if(timeout==null || timeout.getMillis()<=0) return;
1307                
1308                int ms = (int)timeout.getMillis();
1309                if(ms<0)ms=Integer.MAX_VALUE; // long value was bigger than Integer.MAX
1310                
1311        SocketConfig sc=SocketConfig.custom()
1312                        .setSoTimeout(ms)
1313                        .build();
1314        builder.setDefaultSocketConfig(sc);
1315        }
1316
1317        private void parseCookie(Query cookies,String raw) {
1318                String[] arr =ListUtil.trimItems(ListUtil.trim(ListUtil.listToStringArray(raw, ';')));
1319                if(arr.length==0) return;
1320                int row = cookies.addRow();
1321                String item;
1322                
1323                int index;
1324                // name/value
1325                if(arr.length>0) {
1326                        item=arr[0];
1327                        index=item.indexOf('=');
1328                        if(index==-1)  // only name
1329                                cookies.setAtEL(KeyConstants._name,row, dec(item));
1330                        else { // name and value
1331                                cookies.setAtEL(KeyConstants._name,row, dec(item.substring(0,index)));
1332                                cookies.setAtEL(KeyConstants._value,row, dec(item.substring(index+1)));
1333                        }
1334                        
1335                }
1336                String n,v;
1337                cookies.setAtEL("secure",row, Boolean.FALSE);
1338                cookies.setAtEL("httpOnly",row, Boolean.FALSE);
1339                for(int i=1;i<arr.length;i++){
1340                        item=arr[i];
1341                        index=item.indexOf('=');
1342                        if(index==-1)  // only name
1343                                cookies.setAtEL(dec(item),row, Boolean.TRUE);
1344                        else { // name and value
1345                                n=dec(item.substring(0,index));
1346                                v=dec(item.substring(index+1));
1347                                if(n.equalsIgnoreCase("expires")) {
1348                                        DateTime d = Caster.toDate(v, false, null,null);
1349                                        
1350                                        if(d!=null) {
1351                                                cookies.setAtEL(n,row, d);
1352                                                continue;
1353                                        }
1354                                }
1355                                cookies.setAtEL(n,row, v);
1356                        }
1357                        
1358                }
1359        }
1360        
1361        public String dec(String str) {
1362        return ReqRspUtil.decode(str, charset, false);
1363        }
1364
1365        private PageException toPageException(Throwable t, HTTPResponse4Impl rsp) {
1366                if(t instanceof SocketTimeoutException) {
1367                        HTTPException he = new HTTPException("408 Request Time-out","a timeout occurred in tag http",408,"Time-out",rsp==null?null:rsp.getURL());
1368                        List<StackTraceElement> merged = ArrayUtil.merge(t.getStackTrace(), he.getStackTrace());
1369                        StackTraceElement[] traces=new StackTraceElement[merged.size()];
1370                        Iterator<StackTraceElement> it = merged.iterator();
1371                        int index=0;
1372                        while(it.hasNext()){
1373                                traces[index++]=it.next();
1374                        }
1375                        he.setStackTrace(traces);
1376                        return he;
1377                }
1378                PageException pe = Caster.toPageException(t);
1379                if(pe instanceof NativeException) {
1380                        ((NativeException) pe).setAdditional(KeyConstants._url, url);
1381                }
1382                return pe;
1383        }
1384
1385        private void setUnknownHost(Struct cfhttp,Throwable t) {
1386                cfhttp.setEL(ERROR_DETAIL,"Unknown host: "+t.getMessage());
1387                cfhttp.setEL(FILE_CONTENT,"Connection Failure");
1388                cfhttp.setEL(KeyConstants._mimetype,"Unable to determine MIME type of file.");
1389                cfhttp.setEL(STATUSCODE,"Connection Failure. Status code unavailable.");
1390                cfhttp.setEL(RESPONSEHEADER,new StructImpl());
1391                cfhttp.setEL(KeyConstants._text,Boolean.TRUE);
1392                cfhttp.setEL(CHARSET,"");
1393                cfhttp.setEL(KeyConstants._header,"");
1394        }
1395
1396        private void setRequestTimeout(Struct cfhttp) {
1397                cfhttp.setEL(CHARSET,"");
1398                cfhttp.setEL(ERROR_DETAIL,"");
1399                cfhttp.setEL(FILE_CONTENT,"Connection Timeout");
1400                cfhttp.setEL(KeyConstants._header,"");
1401                cfhttp.setEL(KeyConstants._mimetype,"Unable to determine MIME type of file.");
1402                cfhttp.setEL(RESPONSEHEADER,new StructImpl());
1403                cfhttp.setEL(STATUSCODE,"408 Request Time-out");
1404                cfhttp.setEL(STATUS_CODE,new Double(408));
1405                cfhttp.setEL(STATUS_TEXT,"Request Time-out");
1406                cfhttp.setEL(KeyConstants._text,Boolean.TRUE);
1407        }
1408
1409    @Override
1410        public void doInitBody()        {
1411                
1412        }
1413
1414        @Override
1415        public int doAfterBody()        {
1416                return SKIP_BODY;
1417        }
1418
1419        /**
1420         * sets if has body or not
1421         * @param hasBody
1422         */
1423        public void hasBody(boolean hasBody) {
1424            
1425        }
1426
1427        /**
1428         * @param param
1429         */
1430        public void setParam(HttpParamBean param) {
1431                params.add(param);
1432                
1433        }
1434        
1435        
1436    /**
1437     * @param getAsBinary The getasbinary to set.
1438     */
1439    public void setGetasbinary(String getAsBinary) {
1440        // TODO support never, wird das verwendet?
1441        getAsBinary=getAsBinary.toLowerCase().trim();
1442        if(getAsBinary.equals("yes") || getAsBinary.equals("true"))             this.getAsBinary=GET_AS_BINARY_YES;
1443        else if(getAsBinary.equals("no") || getAsBinary.equals("false"))        this.getAsBinary=GET_AS_BINARY_NO;
1444        else if(getAsBinary.equals("auto"))                                                             this.getAsBinary=GET_AS_BINARY_AUTO;
1445    }
1446
1447    /**
1448     * @param multipart The multipart to set.
1449     */
1450    public void setMultipart(boolean multiPart) {
1451        this.multiPart = multiPart;
1452    }
1453
1454    /**
1455     * @param multipart The multipart to set.
1456     * @throws ApplicationException 
1457     */
1458    public void setMultiparttype(String multiPartType) throws ApplicationException {
1459        if(StringUtil.isEmpty(multiPartType))return;
1460        multiPartType=multiPartType.trim().toLowerCase();
1461        
1462        if("form-data".equals(multiPartType))   this.multiPartType=MULTIPART_FORM_DATA;
1463        //else if("related".equals(multiPartType))              this.multiPartType=MultipartRequestEntityFlex.MULTIPART_RELATED;
1464        else
1465                        throw new ApplicationException("invalid value for attribute multiPartType ["+multiPartType+"]",
1466                                        "attribute must have one of the following values [form-data]");
1467                        
1468    }
1469
1470    /**
1471     * @param result The result to set.
1472     */
1473    public void setResult(String result) {
1474        this.result = result;
1475    }
1476
1477        /**
1478         * @param addtoken the addtoken to set
1479         */
1480        public void setAddtoken(boolean addtoken) {
1481                this.addtoken = addtoken;
1482        }
1483}
1484
1485class Executor41 extends Thread {
1486
1487        private final PageContext pc;
1488         final Http41 http;
1489         private final CloseableHttpClient client;
1490         final boolean redirect;
1491         Throwable t;
1492         boolean done;
1493        //URL redirectURL;
1494        HTTPResponse4Impl response;
1495        private HttpRequestBase req;
1496        private HttpContext context;
1497
1498        public Executor41(PageContext pc,Http41 http,CloseableHttpClient client, HttpContext context, HttpRequestBase req, boolean redirect) {
1499                this.pc=pc;
1500                this.http=http;
1501                this.client=client;
1502                this.context=context;
1503                this.redirect=redirect;
1504                this.req=req;
1505        }
1506        
1507        @Override
1508        public void run(){
1509                ThreadLocalPageContext.register(pc);
1510                try {
1511                        response=execute(context);
1512                        done=true;
1513                } 
1514                catch (Throwable t) {
1515                        ExceptionUtil.rethrowIfNecessary(t);
1516                        this.t=t;
1517                }
1518                finally {
1519                        SystemUtil.notify(http);
1520                }
1521        }
1522        
1523        public HTTPResponse4Impl execute(HttpContext context) throws IOException        {
1524                return response=new HTTPResponse4Impl(null,context,req,client.execute(req,context));
1525        }
1526}