Merge branch 'master' of https://github.com/asciimoo/searx

README.md

@@ -13,6 +13,7 @@ List of [running instances](https://github.com/asciimoo/searx/wiki/Searx-instanc
 * Modular (see [examples](https://github.com/asciimoo/searx/blob/master/examples))
 * Parallel queries
 * Supports json output `curl https://searx.0x2a.tk/?format=json&q=[query]`
+* Supports csv output `curl https://searx.0x2a.tk/?format=csv&q=[query]`
 * Opensearch support (you can set as default search engine)
 * Configurable search engines/categories
 * User-agent forwarding
@@ -32,7 +33,6 @@ List of [running instances](https://github.com/asciimoo/searx/wiki/Searx-instanc
 * Language support
 * Documentation
 * Pagination
-* Search suggestions
 * Tests
 
 

engines.cfg_sample

@@ -79,3 +79,8 @@ suggestion_xpath = //div[@id="satat"]//a
 [youtube]
 engine = youtube
 categories = videos
+
+[dailymotion]
+engine = dailymotion
+categories = videos
+

searx/engines/dailymotion.py

@@ -0,0 +1,32 @@
+from urllib import urlencode
+from json import loads
+from cgi import escape
+
+categories = ['videos']
+localization = 'en'
+
+# see http://www.dailymotion.com/doc/api/obj-video.html
+search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'
+
+def request(query, params):
+    global search_url
+    params['url'] = search_url.format(query=urlencode({'search': query, 'localization': localization }))
+    return params
+
+
+def response(resp):
+    results = []
+    search_res = loads(resp.text)
+    if not 'list' in search_res:
+        return results
+    for res in search_res['list']:
+        title = res['title']
+        url = res['url']
+        if res['thumbnail_360_url']:
+            content = '<a href="{0}" title="{0}" ><img src="{1}" /></a><br />'.format(url, res['thumbnail_360_url'])
+        else:
+            content = ''
+        if res['description']:
+            content += escape(res['description'][:500])
+        results.append({'url': url, 'title': title, 'content': content})
+    return results

searx/engines/flickr.py

@@ -7,7 +7,7 @@ from urlparse import urljoin
 categories = ['images']
 
 url = 'https://secure.flickr.com/'
-search_url = url+'search/?q={query}'
+search_url = url+'search/?{query}'
 
 def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}))

searx/engines/piratebay.py

@@ -5,7 +5,7 @@ from urllib import quote
 
 categories = ['videos', 'music']
 
-url = 'https://thepiratebay.sx/'
+url = 'https://thepiratebay.se/'
 search_url = url + 'search/{search_term}/0/99/{search_type}'
 search_types = {'videos': '200'
                ,'music' : '100'

searx/engines/xpath.py

@@ -28,7 +28,7 @@ def extract_url(xpath_results):
             url = xpath_results[0].attrib.get('href')
     else:
         url = xpath_results.attrib.get('href')
-    if not url.startswith('http://') or not url.startswith('https://'):
+    if not url.startswith('http://') and not url.startswith('https://'):
         url = 'http://'+url
     parsed_url = urlparse(url)
     if not parsed_url.netloc:

searx/settings.py

@@ -13,4 +13,4 @@ blacklist = [] # search engine blacklist
 
 categories = {} # custom search engine categories
 
-hostname = None # domain name or None - if you want to rewrite the default HTTP host
+base_url = None # "https://your.domain.tld/" or None (to use request parameters)

searx/static/js/searx.js

@@ -0,0 +1,27 @@
+(function (w, d) {
+    'use strict';
+    function addListener(el, type, fn) {
+        if (el.addEventListener) {
+            el.addEventListener(type, fn, false);
+        } else {
+            el.attachEvent('on' + type, fn);
+        }
+    }
+
+    function placeCursorAtEnd() {
+        if (this.setSelectionRange) {
+            var len = this.value.length * 2;
+            this.setSelectionRange(len, len);
+        }
+    }
+
+    addListener(w, 'load', function () {
+        var qinput = d.getElementById('q');
+        if (qinput !== null) {
+            addListener(qinput, 'focus', placeCursorAtEnd);
+            qinput.focus();
+        }
+    });
+
+})(window, document);
+

searx/templates/base.html

@@ -18,6 +18,7 @@
 <div id="container">
 {% block content %}
 {% endblock %}
+<script src="/static/js/searx.js" ></script>
 </div>
 </body>
 </html>

searx/templates/categories.html

@@ -0,0 +1,5 @@
+{% for category in categories %}
+    <div class="checkbox_container">
+        <input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} /><label for="checkbox_{{ category|replace(' ', '_') }}" class="cb"></label><label for="checkbox_{{ category|replace(' ', '_') }}">{{ category }}</label>
+    </div>
+{% endfor %}

searx/templates/index.html

@@ -6,6 +6,7 @@
     {% include 'search.html' %}
     <p class="top_margin">
         <a href="/about" class="hmarg">about</a>
+        <a href="/preferences" class="hmarg">preferences</a>
     </p>
 </div>
 {% endblock %}

searx/templates/preferences.html

@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+{% block head %} {% endblock %}
+{% block content %}
+<div class="row">
+    <h2>Preferences</h2>
+
+
+    <fieldset>
+        <legend>Default categories</legend>
+        <form method="post" action="/preferences" id="search_form">
+        <p>
+        {% include 'categories.html' %}
+        </p>
+        <input type="submit" value="save" />
+        </form>
+    </fieldset>
+    <div class="right"><a href="/">back</a></div>
+</div>
+{% endblock %}

searx/templates/results.html

@@ -1,12 +1,13 @@
 {% extends "base.html" %}
 {% block title %}{{ q }} - {% endblock %}
 {% block content %}
+<div class="right"><a href="/preferences">preferences</a></div>
 <div class="small">
     {% include 'search.html' %}
 </div>
 <div id="results">
     {% if suggestions %}
-    <div id="suggestions">Suggestions: {% for suggestion in suggestions %}<form method="post" action=""><input type="hidden" name="q" value="{{suggestion}}"><input type="submit" value="{{ suggestion }}" /></form>{% endfor %}</div>
+    <div id="suggestions">Suggestions: {% for suggestion in suggestions %}<form method="post" action="/"><input type="hidden" name="q" value="{{suggestion}}"><input type="submit" value="{{ suggestion }}" /></form>{% endfor %}</div>
     {% endif %}
     <div>
         Number of results: {{ number_of_results }}
@@ -18,14 +19,14 @@
             {% include 'result_templates/default.html' %}
         {% endif %}
     {% endfor %}
-    <form method="post" action="">
+    <form method="post" action="/">
         <div class="left">
             <input type="hidden" name="q" value="{{ q }}" />
             <input type="hidden" name="format" value="csv" />
             <input type="submit" value="download results in csv" />
         </div>
     </form>
-    <form method="post" action="">
+    <form method="post" action="/">
         <div class="">
             <input type="hidden" name="q" value="{{ q }}" />
             <input type="hidden" name="format" value="json" />

searx/templates/search.html

@@ -4,10 +4,6 @@
     <input type="submit" value="" id="search_submit" />
 </div>
     <div>
-    {% for category in categories %}
-        <div class="checkbox_container">
-            <input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} /><label for="checkbox_{{ category|replace(' ', '_') }}" class="cb"></label><label for="checkbox_{{ category|replace(' ', '_') }}">{{ category }}</label>
-        </div>
-    {% endfor %}
+    {% include 'categories.html' %}
     </div>
 </form>

searx/webapp.py

@@ -22,7 +22,7 @@ if __name__ == "__main__":
     from sys import path
     path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))
 
-from flask import Flask, request, render_template, url_for, Response, make_response
+from flask import Flask, request, render_template, url_for, Response, make_response, redirect
 from searx.engines import search, categories, engines, get_engines_stats
 from searx import settings
 import json
@@ -124,29 +124,46 @@ def index():
         response.headers.add('Content-Disposition', 'attachment;Filename=searx_-_{0}.csv'.format('_'.join(query.split())))
         return response
 
-    template = render('results.html'
+    return render('results.html'
                  ,results=results
                  ,q=request_data['q']
                  ,selected_categories=selected_categories
                  ,number_of_results=len(results)
                  ,suggestions=suggestions
                  )
-    resp = make_response(template)
-    resp.set_cookie('categories', ','.join(selected_categories))
 
-    return resp
 
 @app.route('/about', methods=['GET'])
 def about():
     global categories
     return render('about.html', categs=categories.items())
 
+
+@app.route('/preferences', methods=['GET', 'POST'])
+def preferences():
+
+    if request.method=='POST':
+        selected_categories = []
+        for pd_name,pd in request.form.items():
+            if pd_name.startswith('category_'):
+                category = pd_name[9:]
+                if not category in categories:
+                    continue
+                selected_categories.append(category)
+        if selected_categories:
+            resp = make_response(redirect('/'))
+            resp.set_cookie('categories', ','.join(selected_categories))
+            return resp
+    return render('preferences.html')
+
+
 @app.route('/stats', methods=['GET'])
 def stats():
     global categories
     stats = get_engines_stats()
     return render('stats.html', stats=stats)
 
+
 @app.route('/robots.txt', methods=['GET'])
 def robots():
     return Response("""User-agent: *
@@ -155,6 +172,7 @@ Allow: /about
 Disallow: /stats
 """, mimetype='text/plain')
 
+
 @app.route('/opensearch.xml', methods=['GET'])
 def opensearch():
     global opensearch_xml
@@ -165,8 +183,8 @@ def opensearch():
         method = 'get'
     if request.is_secure:
         scheme = 'https'
-    if settings.hostname:
-        hostname = '{0}://{1}/'.format(scheme,settings.hostname)
+    if settings.base_url:
+        hostname = settings.base_url
     else:
         hostname = url_for('index', _external=True, _scheme=scheme)
     ret = opensearch_xml.format(method=method, host=hostname)