From d07cfd9089d05a6a81109518f03ba82660a4aef7 Mon Sep 17 00:00:00 2001
From: dalf <alex@al-f.net>
Date: Wed, 21 Jan 2015 11:33:16 +0100
Subject: [PATCH] [enh] use one single http connection pool : improve response
 time. close #100

---
 searx/autocomplete.py     |  2 +-
 searx/engines/wikidata.py |  2 +-
 searx/poolrequests.py     | 61 +++++++++++++++++++++++++++++++++++++++
 searx/search.py           |  3 +-
 searx/webapp.py           |  2 +-
 5 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 searx/poolrequests.py

diff --git a/searx/autocomplete.py b/searx/autocomplete.py
index 7ebebf1d..e37d34d7 100644
--- a/searx/autocomplete.py
+++ b/searx/autocomplete.py
@@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 
 from lxml import etree
-from requests import get
 from json import loads
 from urllib import urlencode
 from searx.languages import language_codes
 from searx.engines import (
     categories, engines, engine_shortcuts
 )
+from searx.poolrequests import get
 
 
 def searx_bang(full_query):
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index da3152ac..43f72761 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -1,6 +1,6 @@
 import json
-from requests import get
 from urllib import urlencode
+from searx.poolrequests import get
 from searx.utils import format_date_by_locale
 
 result_count = 1
diff --git a/searx/poolrequests.py b/searx/poolrequests.py
new file mode 100644
index 00000000..ef7a195f
--- /dev/null
+++ b/searx/poolrequests.py
@@ -0,0 +1,61 @@
+import requests
+
+
+the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
+the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
+
+
+class SessionSinglePool(requests.Session):
+
+    def __init__(self):
+        global the_https_adapter, the_http_adapter
+        super(SessionSinglePool, self).__init__()
+
+        # reuse the same adapters
+        self.adapters.clear()
+        self.mount('https://', the_https_adapter)
+        self.mount('http://', the_http_adapter)
+
+    def close(self):
+        """Call super, but clear adapters since there are managed globaly"""
+        self.adapters.clear()
+        super(SessionSinglePool, self).close()
+
+
+def request(method, url, **kwargs):
+    """same as requests/requests/api.py request(...) except it use SessionSinglePool"""
+    session = SessionSinglePool()
+    response = session.request(method=method, url=url, **kwargs)
+    session.close()
+    return response
+
+
+def get(url, **kwargs):
+    kwargs.setdefault('allow_redirects', True)
+    return request('get', url, **kwargs)
+
+
+def options(url, **kwargs):
+    kwargs.setdefault('allow_redirects', True)
+    return request('options', url, **kwargs)
+
+
+def head(url, **kwargs):
+    kwargs.setdefault('allow_redirects', False)
+    return request('head', url, **kwargs)
+
+
+def post(url, data=None, json=None, **kwargs):
+    return request('post', url, data=data, json=json, **kwargs)
+
+
+def put(url, data=None, **kwargs):
+    return request('put', url, data=data, **kwargs)
+
+
+def patch(url, data=None, **kwargs):
+    return request('patch', url, data=data, **kwargs)
+
+
+def delete(url, **kwargs):
+    return request('delete', url, **kwargs)
diff --git a/searx/search.py b/searx/search.py
index 0324d4aa..b6cf84e9 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-import requests as requests_lib
 import threading
 import re
+import searx.poolrequests as requests_lib
 from itertools import izip_longest, chain
 from operator import itemgetter
 from Queue import Queue
@@ -31,7 +31,6 @@ from searx.utils import gen_useragent
 from searx.query import Query
 from searx import logger
 
-
 logger = logger.getChild('search')
 
 number_of_searches = 0
diff --git a/searx/webapp.py b/searx/webapp.py
index 41bad5ea..53576b4d 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -27,7 +27,6 @@ import cStringIO
 import os
 
 from datetime import datetime, timedelta
-from requests import get as http_get
 from itertools import chain
 from urllib import urlencode
 from flask import (
@@ -36,6 +35,7 @@ from flask import (
 )
 from flask.ext.babel import Babel, gettext, format_date
 from searx import settings, searx_dir
+from searx.poolrequests import get as http_get
 from searx.engines import (
     categories, engines, get_engines_stats, engine_shortcuts
 )