add otg indexing support

.gitignore

@@ -2,3 +2,4 @@
 .env
 __pycache__/
 venv/
+*.sh

README.md

@@ -61,7 +61,13 @@ This is the general format, change the values of corresponding fields as your re
     "index_group": false,
     "index_channel": true,
     "exclude_chats": [],
-    "include_chats": []
+    "include_chats": [],
+    "otg": {
+        "enable": false,
+        "include_private": false,
+        "include_group": true,
+        "include_channel": true
+    }
 }
 ```
 > * `index_all` - Whether to consider all the chats associated with the telegram account. Value should either be `true` or `false`.
@@ -70,6 +76,7 @@ This is the general format, change the values of corresponding fields as your re
 > * `index_channel` - Whether to index channels. Only considered if `index_all` is set to `true`. Value should either be `true` or `false`.
 > * `exclude_chats` - An array/list of chat id's that should be ignored for indexing. Only considered if `index_all` is set to `true`.
 > * `include_chats` - An array/list of chat id's to index. Only considered if `index_all` is set to `false`.
+> * `otg` - On-The-Go Indexing settings. Whether to allow indexing channels/chats other than the specified chats dynamically on the go.
 
 
 * **Run app.**

app/config.py

@@ -20,14 +20,12 @@ except (KeyError, ValueError):
     print("\n\nPlease set the API_ID and API_HASH environment variables correctly")
     print("You can get your own API keys at https://my.telegram.org/apps")
     sys.exit(1)
-    
+
 try:
     index_settings_str = os.environ["INDEX_SETTINGS"].strip()
     index_settings = json.loads(index_settings_str)
-    '''
+    otg_settings = index_settings['otg']
-    {"index_all": true, "index_private":false, "index_group": false, "index_channel": true, "include_chats": [], "exclude_chats": []}
+    enable_otg = otg_settings['enable']
-    
-    '''
 except:
     traceback.print_exc()
     print("\n\nPlease set the INDEX_SETTINGS environment variable correctly")

app/routes.py

@@ -29,9 +29,18 @@ def generate_alias_id(chat):
 async def setup_routes(app, handler):
     h = handler
     client = h.client
+    p = r"/{chat:[^/]+}"
     routes =  [
         web.get('/', h.home),
-        web.get('/api', h.api_home),
+        web.post('/otg', h.dynamic_view),
+        web.get('/otg', h.otg_view),
+        web.get(p, h.index),
+        web.get(p + r"/logo", h.logo),
+        web.get(p + r"/{id:\d+}/view", h.info),
+        web.get(p + r"/{id:\d+}/download", h.download_get),
+        web.head(p + r"/{id:\d+}/download", h.download_head),
+        web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
+        web.view(r'/{wildcard:.*}', h.wildcard)
     ]
     index_all = index_settings['index_all']
     index_private = index_settings['index_private']
@@ -44,51 +53,25 @@ async def setup_routes(app, handler):
             alias_id = None
             if chat.id in exclude_chats:
                 continue
-            
+
             if chat.is_user:
                 if index_private:
                     alias_id = generate_alias_id(chat)
-            elif chat.is_group:
+            elif chat.is_channel:
-                if index_group:
-                    alias_id = generate_alias_id(chat)
-            else:
                 if index_channel:
                     alias_id = generate_alias_id(chat)
-            
+            else:
+                if index_group:
+                    alias_id = generate_alias_id(chat)
+
             if not alias_id:
                 continue
-            
-            p = r"/{chat:" + alias_id + "}"
-            p_api = '/api' + p
-            r = [
-                web.get(p, h.index),
-                web.get(p_api, h.api_index),
-                web.get(p + r"/logo", h.logo),
-                web.get(p + r"/{id:\d+}/view", h.info),
-                web.get(p_api + r"/{id:\d+}/view", h.api_info),
-                web.get(p + r"/{id:\d+}/download", h.download_get),
-                web.head(p + r"/{id:\d+}/download", h.download_head),
-                web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
-            ]
-            routes += r
             log.debug(f"Index added for {chat.id} :: {chat.title} at /{alias_id}")
+
     else:
         for chat_id in include_chats:
             chat = await client.get_entity(chat_id)
             alias_id = generate_alias_id(chat)
-            p = r"/{chat:" + alias_id + "}"
-            p_api = '/api' + p
-            r = [
-                web.get(p, h.index),
-                web.get(p_api, h.api_index),
-                web.get(p + r"/logo", h.logo),
-                web.get(p + r"/{id:\d+}/view", h.info),
-                web.get(p_api + r"/{id:\d+}/view", h.api_info),
-                web.get(p + r"/{id:\d+}/download", h.download_get),
-                web.head(p + r"/{id:\d+}/download", h.download_head),
-                web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
-            ]
-            routes += r
             log.debug(f"Index added for {chat.id} :: {chat.title} at /{alias_id}")
-    routes.append(web.view(r'/{wildcard:.*}', h.wildcard))
+
     app.add_routes(routes)

app/templates/header.html

@@ -2,19 +2,22 @@
 <html lang="en">
     <head>
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
-        
+
         <link href="https://unpkg.com/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">
         <script src="https://cdn.fluidplayer.com/v3/current/fluidplayer.min.js"></script>
-        
+
         <title>
             {% if title %} {{title}} {% else %} Telegram Index {% endif %}
         </title>
-        
+
     </head>
     <body>
-        
+
         <div class="m-auto w-full xl:max-w-screen-xl min-h-screen">
 
-            <header class="bg-red-600 text-white mb-2 p-4">
+            <header class="flex justify-between bg-red-600 text-white mb-2 p-4 w-full sticky top-0 shadow">
                 <a href="/" class="text-left text-xl lg:text-2xl xl:text-3xl"> Telegram Index </a>
+                {% if otg %}
+                    <a title="On-The-Go Indexing" href="/otg" class="text-xl lg:text-2xl xl:text-3xl"> OTG Indexing </a>
+                {% endif %}
             </header>

app/templates/home.html

@@ -1,16 +1,17 @@
 {% include 'header.html' %}
-    
+
     <h1 class=" my-2 text-2xl text-center font-bold text-green-400">
         Available Sources
     </h1>
+
     <div class="mx-auto my-2 p-2 w-full">
         <div class="block p-4 md:flex md:flex-wrap md:justify-center w-full text-center break-words">
             {% for chat in chats %}
                 <a title="{{chat.name}}" href="/{{chat.page_id}}" class="flex flex-col justify-around w-full min-h-full md:w-2/5 lg:w-1/5 rounded my-2 md:mx-1 shadow-md hover:shadow-lg hover:border hover:border-blue-300 hover:bg-blue-100">
-                
+
                     <img src="/{{chat.page_id}}/logo?big=1" class="w-full rounded-full ">
                     <div class="p-4 rounded">{{chat.name}}</div>
-                
+
                 </a>
             {% endfor %}
         </div>

app/templates/otg.html

@@ -0,0 +1,34 @@
+{% include 'header.html' %}
+
+    <h1 class="px-4 my-2 text-2xl text-center font-bold text-green-400">
+        On-The-Go Indexing
+    </h1>
+
+    <div class="px-4 mx-auto my-2">
+
+        <p class="text-center text-gray-700">
+            On-The-Go (OTG) Indexing. Index any public telegram channel or public telegram group on the go with it's username.
+        </p>
+
+    </div>
+
+    {% if error %}
+        <div id="alert" class="px-4 mx-auto my-2 text-center m-2 ">
+            <p class="text-white bg-red-500 w-auto inline-block px-4 py-2 rounded-lg"> {{error}} </p>
+        </div>
+        <script>
+        setTimeout(function(){
+            document.getElementById("alert").style.display = 'none';
+        }, 7000);
+        </script>
+    {% endif %}
+
+    <div class="px-4 mx-auto my-2">
+        <form class="my-2 text-center flex justify-center" action="/otg" method="POST">
+                <input class="px-4 py-2 rounded-l-full border border-r-0 border-teal-400 outline-none" type="text" name="id" placeholder="Enter username..." required>
+                <button class="px-4 py-2 bg-teal-400 border border-teal-400 hover:bg-white hover:text-teal-400 text-white rounded-r-full outline-none" type="submit">Get Result</button>
+        </form>
+    </div>
+
+
+{% include 'footer.html' %}

app/views.py

@@ -8,21 +8,30 @@ import aiohttp_jinja2
 from jinja2 import Markup
 from telethon.tl import types
 from telethon.tl.custom import Message
+from telethon.tl.types import User, Chat, Channel
 
 from .util import get_file_name, get_human_size
-from .config import index_settings, chat_ids
+from .config import otg_settings, chat_ids, enable_otg
 
 
 log = logging.getLogger(__name__)
 
 
 class Views:
-    
+
     def __init__(self, client):
         self.client = client
-    
+
-    
+
-    async def _home(self, req):
+    async def wildcard(self, req):
+        raise web.HTTPFound('/')
+
+
+    @aiohttp_jinja2.template('home.html')
+    async def home(self, req):
+        if len(chat_ids) == 1:
+            raise web.HTTPFound(f"{chat_ids[0]['alias_id']}")
+
         chats = []
         for chat in chat_ids:
             chats.append({
@@ -30,40 +39,71 @@ class Views:
                 'name': chat['title'],
                 'url': req.rel_url.path + f"/{chat['alias_id']}"
             })
-        return {'chats':chats}
+        return {'chats':chats, 'otg': enable_otg}
-    
-    
-    async def wildcard(self, req):
-        raise web.HTTPFound('/')
-    
 
-    @aiohttp_jinja2.template('home.html')
+
-    async def home(self, req):
+    @aiohttp_jinja2.template('otg.html')
-        if len(chat_ids) == 1:
+    async def otg_view(self, req):
-            raise web.HTTPFound(f"{chat_ids[0]['alias_id']}")
+        if not enable_otg:
-        return await self._home(req)
+            raise web.HTTPFound('/')
-    
+        return_data = {}
-    
+        error = req.query.get('e')
-    async def api_home(self, req):
+        if error:
-        data = await self._home(req)
+            return_data.update({'error': error})
-        return web.json_response(data)
+
+        return return_data
+
+
+
+    async def dynamic_view(self, req):
+        if not enable_otg:
+            raise web.HTTPFound('/')
+
+        rel_url = req.rel_url
+        include_private = otg_settings['include_private']
+        include_group = otg_settings['include_group']
+        include_channel = otg_settings['include_channel']
+        post_data = await req.post()
+        raw_id = post_data.get('id')
+        if not raw_id:
+            raise web.HTTPFound('/')
+
+        raw_id.replace('@', '')
+        try:
+            chat = await self.client.get_entity(raw_id)
+        except Exception as e:
+            log.debug(e, exc_info=True)
+            raise web.HTTPFound(rel_url.with_query({'e': f"No chat found with username {raw_id}"}))
+
+        if isinstance(chat, User) and not include_private:
+            raise web.HTTPFound(rel_url.with_query({'e': "Indexing private chats is not supported!!"}))
+        elif isinstance(chat, Channel) and not include_channel:
+            raise web.HTTPFound(rel_url.with_query({'e': "Indexing channels is not supported!!"}))
+        elif isinstance(chat, Chat) and not include_group:
+            raise web.HTTPFound(rel_url.with_query({'e': "Indexing group chats is not supported!!"}))
+
+        log.debug(f"chat {chat} accessed!!")
+        raise web.HTTPFound(f'/{chat.id}')
 
 
     @aiohttp_jinja2.template('index.html')
     async def index(self, req):
-        return await self._index(req)
-    
-    
-    async def api_index(self, req):
-        data = await self._index(req, True)
-        return web.json_response(data)
-    
-    
-    async def _index(self, req, api=False):
         alias_id = req.match_info['chat']
-        chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
+        chat = [i for i in chat_ids if i['alias_id'] == alias_id]
-        chat_id = chat['chat_id']
+        if not chat:
-        chat_name = chat['title']
+            if not enable_otg:
+                raise web.HTTPFound('/')
+
+            try:
+                chat_id = int(alias_id)
+                chat_ = await self.client.get_entity(chat_id)
+                chat_name = chat_.title
+            except:
+                raise web.HTTPFound('/')
+        else:
+            chat = chat[0]
+            chat_id = chat['chat_id']
+            chat_name = chat['title']
         log_msg = ''
         try:
             offset_val = int(req.query.get('page', '1'))
@@ -85,7 +125,7 @@ class Views:
             if search_query:
                 kwargs.update({'search': search_query})
             messages = (await self.client.get_messages(**kwargs)) or []
-            
+
         except:
             log.debug("failed to get messages", exc_info=True)
             messages = []
@@ -129,7 +169,7 @@ class Views:
                 'url': str(req.rel_url.with_query(query)),
                 'no': offset_val
             }
-        
+
         if len(messages)==20:
             query = {'page':offset_val+2}
             if search_query:
@@ -140,12 +180,12 @@ class Views:
             }
 
         return {
-            'item_list':results, 
+            'item_list':results,
             'prev_page': prev_page,
             'cur_page' : offset_val+1,
             'next_page': next_page,
             'search': search_query,
-            'name' : chat['title'],
+            'name' : chat_name,
             'logo': f"/{alias_id}/logo",
             'title' : "Index of " + chat_name
         }
@@ -153,22 +193,19 @@ class Views:
 
     @aiohttp_jinja2.template('info.html')
     async def info(self, req):
-        return await self._info(req)
-    
-    
-    async def api_info(self, req):
-        data = await self._info(req)
-        if not data['found']:
-            return web.Response(status=404, text="404: Not Found")
-        
-        return web.json_response(data)
-        
-        
-    async def _info(self, req):
         file_id = int(req.match_info["id"])
         alias_id = req.match_info['chat']
-        chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
+        chat = [i for i in chat_ids if i['alias_id'] == alias_id]
-        chat_id = chat['chat_id']
+        if not chat:
+            if not enable_otg:
+                raise web.HTTPFound('/')
+            try:
+                chat_id = int(alias_id)
+            except:
+                raise web.HTTPFound('/')
+        else:
+            chat = chat[0]
+            chat_id = chat['chat_id']
         try:
             message = await self.client.get_messages(entity=chat_id, ids=file_id)
         except:
@@ -205,7 +242,7 @@ class Views:
                 media['audio'] = True
             elif 'image/' in message.file.mime_type:
                 media['image'] = True
-                
+
             if message.text:
                 caption = message.raw_text
             else:
@@ -244,12 +281,21 @@ class Views:
             }
         log.debug(f"data for {file_id} in {chat_id} returned as {return_val}")
         return return_val
-    
+
 
     async def logo(self, req):
         alias_id = req.match_info['chat']
-        chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
+        chat = [i for i in chat_ids if i['alias_id'] == alias_id]
-        chat_id = chat['chat_id']
+        if not chat:
+            if not enable_otg:
+                return web.Response(status=403, text="403: Forbiden")
+            try:
+                chat_id = int(alias_id)
+            except:
+                return web.Response(status=403, text="403: Forbiden")
+        else:
+            chat = chat[0]
+            chat_id = chat['chat_id']
         chat_name = "Image not available"
         try:
             photo = await self.client.get_profile_photos(chat_id)
@@ -281,7 +327,7 @@ class Views:
                     thumb_size=size.type
                 )
                 body = self.client.iter_download(media)
-        
+
         r = web.Response(
             status=200,
             body=body,
@@ -292,31 +338,40 @@ class Views:
         )
         #r.enable_chunked_encoding()
         return r
-    
+
-    
+
     async def download_get(self, req):
         return await self.handle_request(req)
-    
+
-    
+
     async def download_head(self, req):
         return await self.handle_request(req, head=True)
-    
+
-    
+
     async def thumbnail_get(self, req):
         file_id = int(req.match_info["id"])
         alias_id = req.match_info['chat']
-        chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
+        chat = [i for i in chat_ids if i['alias_id'] == alias_id]
-        chat_id = chat['chat_id']
+        if not chat:
+            if not enable_otg:
+                return web.Response(status=403, text="403: Forbiden")
+            try:
+                chat_id = int(alias_id)
+            except:
+                return web.Response(status=403, text="403: Forbiden")
+        else:
+            chat = chat[0]
+            chat_id = chat['chat_id']
         try:
             message = await self.client.get_messages(entity=chat_id, ids=file_id)
         except:
             log.debug(f"Error in getting message {file_id} in {chat_id}", exc_info=True)
             message = None
-        
+
         if not message or not message.file:
             log.debug(f"no result for {file_id} in {chat_id}")
             return web.Response(status=410, text="410: Gone. Access to the target resource is no longer available!")
-        
+
         if message.document:
             media = message.document
             thumbnails = media.thumbs
@@ -325,7 +380,7 @@ class Views:
             media = message.photo
             thumbnails = media.sizes
             location = types.InputPhotoFileLocation
-        
+
         if not thumbnails:
             c = lambda : random.randint(0, 255)
             color = tuple([c() for i in range(3)])
@@ -338,7 +393,7 @@ class Views:
             thumbnail = self.client._get_thumb(thumbnails, thumb_pos)
             if not thumbnail or isinstance(thumbnail, types.PhotoSizeEmpty):
                 return web.Response(status=410, text="410: Gone. Access to the target resource is no longer available!")
-            
+
             if isinstance(thumbnail, (types.PhotoCachedSize, types.PhotoStrippedSize)):
                 body = self.client._download_cached_photo_size(thumbnail, bytes)
             else:
@@ -348,9 +403,9 @@ class Views:
                     file_reference=media.file_reference,
                     thumb_size=thumbnail.type
                 )
-            
+
                 body = self.client.iter_download(actual_file)
-        
+
         r = web.Response(
             status=200,
             body=body,
@@ -361,29 +416,38 @@ class Views:
         )
         #r.enable_chunked_encoding()
         return r
-    
+
 
     async def handle_request(self, req, head=False):
         file_id = int(req.match_info["id"])
         alias_id = req.match_info['chat']
-        chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
+        chat = [i for i in chat_ids if i['alias_id'] == alias_id]
-        chat_id = chat['chat_id']
+        if not chat:
-        
+            if not enable_otg:
+                return web.Response(status=403, text="403: Forbiden")
+            try:
+                chat_id = int(alias_id)
+            except:
+                return web.Response(status=403, text="403: Forbiden")
+        else:
+            chat = chat[0]
+            chat_id = chat['chat_id']
+
         try:
             message = await self.client.get_messages(entity=chat_id, ids=file_id)
         except:
             log.debug(f"Error in getting message {file_id} in {chat_id}", exc_info=True)
             message = None
-        
+
         if not message or not message.file:
             log.debug(f"no result for {file_id} in {chat_id}")
             return web.Response(status=410, text="410: Gone. Access to the target resource is no longer available!")
-        
+
         media = message.media
         size = message.file.size
         file_name = get_file_name(message)
         mime_type = message.file.mime_type
-        
+
         try:
             offset = req.http_range.start or 0
             limit = req.http_range.stop or size
@@ -397,13 +461,13 @@ class Views:
                     "Content-Range": f"bytes */{size}"
                 }
             )
-        
+
         if not head:
             body = self.client.download(media, size, offset, limit)
             log.info(f"Serving file in {message.id} (chat {chat_id}) ; Range: {offset} - {limit}")
         else:
             body = None
-        
+
         headers = {
             "Content-Type": mime_type,
             "Content-Range": f"bytes {offset}-{limit}/{size}",