add otg indexing support

This commit is contained in:
odysseusmax 2020-09-20 22:08:52 +05:30
parent fe194ca9e8
commit 2fffd9353d
8 changed files with 219 additions and 128 deletions

1
.gitignore vendored
View File

@ -2,3 +2,4 @@
.env
__pycache__/
venv/
*.sh

View File

@ -61,7 +61,13 @@ This is the general format, change the values of corresponding fields as your re
"index_group": false,
"index_channel": true,
"exclude_chats": [],
"include_chats": []
"include_chats": [],
"otg": {
"enable": false,
"include_private": false,
"include_group": true,
"include_channel": true
}
}
```
> * `index_all` - Whether to consider all the chats associated with the telegram account. Value should either be `true` or `false`.
@ -70,6 +76,7 @@ This is the general format, change the values of corresponding fields as your re
> * `index_channel` - Whether to index channels. Only considered if `index_all` is set to `true`. Value should either be `true` or `false`.
> * `exclude_chats` - An array/list of chat id's that should be ignored for indexing. Only considered if `index_all` is set to `true`.
> * `include_chats` - An array/list of chat id's to index. Only considered if `index_all` is set to `false`.
> * `otg` - On-The-Go Indexing settings. Whether to allow indexing channels/chats other than the specified chats dynamically on the go.
* **Run app.**

View File

@ -20,14 +20,12 @@ except (KeyError, ValueError):
print("\n\nPlease set the API_ID and API_HASH environment variables correctly")
print("You can get your own API keys at https://my.telegram.org/apps")
sys.exit(1)
try:
index_settings_str = os.environ["INDEX_SETTINGS"].strip()
index_settings = json.loads(index_settings_str)
'''
{"index_all": true, "index_private":false, "index_group": false, "index_channel": true, "include_chats": [], "exclude_chats": []}
'''
otg_settings = index_settings['otg']
enable_otg = otg_settings['enable']
except:
traceback.print_exc()
print("\n\nPlease set the INDEX_SETTINGS environment variable correctly")

View File

@ -29,9 +29,18 @@ def generate_alias_id(chat):
async def setup_routes(app, handler):
h = handler
client = h.client
p = r"/{chat:[^/]+}"
routes = [
web.get('/', h.home),
web.get('/api', h.api_home),
web.post('/otg', h.dynamic_view),
web.get('/otg', h.otg_view),
web.get(p, h.index),
web.get(p + r"/logo", h.logo),
web.get(p + r"/{id:\d+}/view", h.info),
web.get(p + r"/{id:\d+}/download", h.download_get),
web.head(p + r"/{id:\d+}/download", h.download_head),
web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
web.view(r'/{wildcard:.*}', h.wildcard)
]
index_all = index_settings['index_all']
index_private = index_settings['index_private']
@ -44,51 +53,25 @@ async def setup_routes(app, handler):
alias_id = None
if chat.id in exclude_chats:
continue
if chat.is_user:
if index_private:
alias_id = generate_alias_id(chat)
elif chat.is_group:
if index_group:
alias_id = generate_alias_id(chat)
else:
elif chat.is_channel:
if index_channel:
alias_id = generate_alias_id(chat)
else:
if index_group:
alias_id = generate_alias_id(chat)
if not alias_id:
continue
p = r"/{chat:" + alias_id + "}"
p_api = '/api' + p
r = [
web.get(p, h.index),
web.get(p_api, h.api_index),
web.get(p + r"/logo", h.logo),
web.get(p + r"/{id:\d+}/view", h.info),
web.get(p_api + r"/{id:\d+}/view", h.api_info),
web.get(p + r"/{id:\d+}/download", h.download_get),
web.head(p + r"/{id:\d+}/download", h.download_head),
web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
]
routes += r
log.debug(f"Index added for {chat.id} :: {chat.title} at /{alias_id}")
else:
for chat_id in include_chats:
chat = await client.get_entity(chat_id)
alias_id = generate_alias_id(chat)
p = r"/{chat:" + alias_id + "}"
p_api = '/api' + p
r = [
web.get(p, h.index),
web.get(p_api, h.api_index),
web.get(p + r"/logo", h.logo),
web.get(p + r"/{id:\d+}/view", h.info),
web.get(p_api + r"/{id:\d+}/view", h.api_info),
web.get(p + r"/{id:\d+}/download", h.download_get),
web.head(p + r"/{id:\d+}/download", h.download_head),
web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
]
routes += r
log.debug(f"Index added for {chat.id} :: {chat.title} at /{alias_id}")
routes.append(web.view(r'/{wildcard:.*}', h.wildcard))
app.add_routes(routes)

View File

@ -2,19 +2,22 @@
<html lang="en">
<head>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link href="https://unpkg.com/tailwindcss@^1.0/dist/tailwind.min.css" rel="stylesheet">
<script src="https://cdn.fluidplayer.com/v3/current/fluidplayer.min.js"></script>
<title>
{% if title %} {{title}} {% else %} Telegram Index {% endif %}
</title>
</head>
<body>
<div class="m-auto w-full xl:max-w-screen-xl min-h-screen">
<header class="bg-red-600 text-white mb-2 p-4">
<header class="flex justify-between bg-red-600 text-white mb-2 p-4 w-full sticky top-0 shadow">
<a href="/" class="text-left text-xl lg:text-2xl xl:text-3xl"> Telegram Index </a>
{% if otg %}
<a title="On-The-Go Indexing" href="/otg" class="text-xl lg:text-2xl xl:text-3xl"> OTG Indexing </a>
{% endif %}
</header>

View File

@ -1,16 +1,17 @@
{% include 'header.html' %}
<h1 class=" my-2 text-2xl text-center font-bold text-green-400">
Available Sources
</h1>
<div class="mx-auto my-2 p-2 w-full">
<div class="block p-4 md:flex md:flex-wrap md:justify-center w-full text-center break-words">
{% for chat in chats %}
<a title="{{chat.name}}" href="/{{chat.page_id}}" class="flex flex-col justify-around w-full min-h-full md:w-2/5 lg:w-1/5 rounded my-2 md:mx-1 shadow-md hover:shadow-lg hover:border hover:border-blue-300 hover:bg-blue-100">
<img src="/{{chat.page_id}}/logo?big=1" class="w-full rounded-full ">
<div class="p-4 rounded">{{chat.name}}</div>
</a>
{% endfor %}
</div>

34
app/templates/otg.html Normal file
View File

@ -0,0 +1,34 @@
{% include 'header.html' %}
<h1 class="px-4 my-2 text-2xl text-center font-bold text-green-400">
On-The-Go Indexing
</h1>
<div class="px-4 mx-auto my-2">
<p class="text-center text-gray-700">
On-The-Go (OTG) Indexing. Index any public telegram channel or public telegram group on the go with it's username.
</p>
</div>
{% if error %}
<div id="alert" class="px-4 mx-auto my-2 text-center m-2 ">
<p class="text-white bg-red-500 w-auto inline-block px-4 py-2 rounded-lg"> {{error}} </p>
</div>
<script>
setTimeout(function(){
document.getElementById("alert").style.display = 'none';
}, 7000);
</script>
{% endif %}
<div class="px-4 mx-auto my-2">
<form class="my-2 text-center flex justify-center" action="/otg" method="POST">
<input class="px-4 py-2 rounded-l-full border border-r-0 border-teal-400 outline-none" type="text" name="id" placeholder="Enter username..." required>
<button class="px-4 py-2 bg-teal-400 border border-teal-400 hover:bg-white hover:text-teal-400 text-white rounded-r-full outline-none" type="submit">Get Result</button>
</form>
</div>
{% include 'footer.html' %}

View File

@ -8,21 +8,30 @@ import aiohttp_jinja2
from jinja2 import Markup
from telethon.tl import types
from telethon.tl.custom import Message
from telethon.tl.types import User, Chat, Channel
from .util import get_file_name, get_human_size
from .config import index_settings, chat_ids
from .config import otg_settings, chat_ids, enable_otg
log = logging.getLogger(__name__)
class Views:
def __init__(self, client):
self.client = client
async def _home(self, req):
async def wildcard(self, req):
raise web.HTTPFound('/')
@aiohttp_jinja2.template('home.html')
async def home(self, req):
if len(chat_ids) == 1:
raise web.HTTPFound(f"{chat_ids[0]['alias_id']}")
chats = []
for chat in chat_ids:
chats.append({
@ -30,40 +39,71 @@ class Views:
'name': chat['title'],
'url': req.rel_url.path + f"/{chat['alias_id']}"
})
return {'chats':chats}
async def wildcard(self, req):
raise web.HTTPFound('/')
return {'chats':chats, 'otg': enable_otg}
@aiohttp_jinja2.template('home.html')
async def home(self, req):
if len(chat_ids) == 1:
raise web.HTTPFound(f"{chat_ids[0]['alias_id']}")
return await self._home(req)
async def api_home(self, req):
data = await self._home(req)
return web.json_response(data)
@aiohttp_jinja2.template('otg.html')
async def otg_view(self, req):
if not enable_otg:
raise web.HTTPFound('/')
return_data = {}
error = req.query.get('e')
if error:
return_data.update({'error': error})
return return_data
async def dynamic_view(self, req):
if not enable_otg:
raise web.HTTPFound('/')
rel_url = req.rel_url
include_private = otg_settings['include_private']
include_group = otg_settings['include_group']
include_channel = otg_settings['include_channel']
post_data = await req.post()
raw_id = post_data.get('id')
if not raw_id:
raise web.HTTPFound('/')
raw_id.replace('@', '')
try:
chat = await self.client.get_entity(raw_id)
except Exception as e:
log.debug(e, exc_info=True)
raise web.HTTPFound(rel_url.with_query({'e': f"No chat found with username {raw_id}"}))
if isinstance(chat, User) and not include_private:
raise web.HTTPFound(rel_url.with_query({'e': "Indexing private chats is not supported!!"}))
elif isinstance(chat, Channel) and not include_channel:
raise web.HTTPFound(rel_url.with_query({'e': "Indexing channels is not supported!!"}))
elif isinstance(chat, Chat) and not include_group:
raise web.HTTPFound(rel_url.with_query({'e': "Indexing group chats is not supported!!"}))
log.debug(f"chat {chat} accessed!!")
raise web.HTTPFound(f'/{chat.id}')
@aiohttp_jinja2.template('index.html')
async def index(self, req):
return await self._index(req)
async def api_index(self, req):
data = await self._index(req, True)
return web.json_response(data)
async def _index(self, req, api=False):
alias_id = req.match_info['chat']
chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
chat_id = chat['chat_id']
chat_name = chat['title']
chat = [i for i in chat_ids if i['alias_id'] == alias_id]
if not chat:
if not enable_otg:
raise web.HTTPFound('/')
try:
chat_id = int(alias_id)
chat_ = await self.client.get_entity(chat_id)
chat_name = chat_.title
except:
raise web.HTTPFound('/')
else:
chat = chat[0]
chat_id = chat['chat_id']
chat_name = chat['title']
log_msg = ''
try:
offset_val = int(req.query.get('page', '1'))
@ -85,7 +125,7 @@ class Views:
if search_query:
kwargs.update({'search': search_query})
messages = (await self.client.get_messages(**kwargs)) or []
except:
log.debug("failed to get messages", exc_info=True)
messages = []
@ -129,7 +169,7 @@ class Views:
'url': str(req.rel_url.with_query(query)),
'no': offset_val
}
if len(messages)==20:
query = {'page':offset_val+2}
if search_query:
@ -140,12 +180,12 @@ class Views:
}
return {
'item_list':results,
'item_list':results,
'prev_page': prev_page,
'cur_page' : offset_val+1,
'next_page': next_page,
'search': search_query,
'name' : chat['title'],
'name' : chat_name,
'logo': f"/{alias_id}/logo",
'title' : "Index of " + chat_name
}
@ -153,22 +193,19 @@ class Views:
@aiohttp_jinja2.template('info.html')
async def info(self, req):
return await self._info(req)
async def api_info(self, req):
data = await self._info(req)
if not data['found']:
return web.Response(status=404, text="404: Not Found")
return web.json_response(data)
async def _info(self, req):
file_id = int(req.match_info["id"])
alias_id = req.match_info['chat']
chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
chat_id = chat['chat_id']
chat = [i for i in chat_ids if i['alias_id'] == alias_id]
if not chat:
if not enable_otg:
raise web.HTTPFound('/')
try:
chat_id = int(alias_id)
except:
raise web.HTTPFound('/')
else:
chat = chat[0]
chat_id = chat['chat_id']
try:
message = await self.client.get_messages(entity=chat_id, ids=file_id)
except:
@ -205,7 +242,7 @@ class Views:
media['audio'] = True
elif 'image/' in message.file.mime_type:
media['image'] = True
if message.text:
caption = message.raw_text
else:
@ -244,12 +281,21 @@ class Views:
}
log.debug(f"data for {file_id} in {chat_id} returned as {return_val}")
return return_val
async def logo(self, req):
alias_id = req.match_info['chat']
chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
chat_id = chat['chat_id']
chat = [i for i in chat_ids if i['alias_id'] == alias_id]
if not chat:
if not enable_otg:
return web.Response(status=403, text="403: Forbiden")
try:
chat_id = int(alias_id)
except:
return web.Response(status=403, text="403: Forbiden")
else:
chat = chat[0]
chat_id = chat['chat_id']
chat_name = "Image not available"
try:
photo = await self.client.get_profile_photos(chat_id)
@ -281,7 +327,7 @@ class Views:
thumb_size=size.type
)
body = self.client.iter_download(media)
r = web.Response(
status=200,
body=body,
@ -292,31 +338,40 @@ class Views:
)
#r.enable_chunked_encoding()
return r
async def download_get(self, req):
return await self.handle_request(req)
async def download_head(self, req):
return await self.handle_request(req, head=True)
async def thumbnail_get(self, req):
file_id = int(req.match_info["id"])
alias_id = req.match_info['chat']
chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
chat_id = chat['chat_id']
chat = [i for i in chat_ids if i['alias_id'] == alias_id]
if not chat:
if not enable_otg:
return web.Response(status=403, text="403: Forbiden")
try:
chat_id = int(alias_id)
except:
return web.Response(status=403, text="403: Forbiden")
else:
chat = chat[0]
chat_id = chat['chat_id']
try:
message = await self.client.get_messages(entity=chat_id, ids=file_id)
except:
log.debug(f"Error in getting message {file_id} in {chat_id}", exc_info=True)
message = None
if not message or not message.file:
log.debug(f"no result for {file_id} in {chat_id}")
return web.Response(status=410, text="410: Gone. Access to the target resource is no longer available!")
if message.document:
media = message.document
thumbnails = media.thumbs
@ -325,7 +380,7 @@ class Views:
media = message.photo
thumbnails = media.sizes
location = types.InputPhotoFileLocation
if not thumbnails:
c = lambda : random.randint(0, 255)
color = tuple([c() for i in range(3)])
@ -338,7 +393,7 @@ class Views:
thumbnail = self.client._get_thumb(thumbnails, thumb_pos)
if not thumbnail or isinstance(thumbnail, types.PhotoSizeEmpty):
return web.Response(status=410, text="410: Gone. Access to the target resource is no longer available!")
if isinstance(thumbnail, (types.PhotoCachedSize, types.PhotoStrippedSize)):
body = self.client._download_cached_photo_size(thumbnail, bytes)
else:
@ -348,9 +403,9 @@ class Views:
file_reference=media.file_reference,
thumb_size=thumbnail.type
)
body = self.client.iter_download(actual_file)
r = web.Response(
status=200,
body=body,
@ -361,29 +416,38 @@ class Views:
)
#r.enable_chunked_encoding()
return r
async def handle_request(self, req, head=False):
file_id = int(req.match_info["id"])
alias_id = req.match_info['chat']
chat = [i for i in chat_ids if i['alias_id'] == alias_id][0]
chat_id = chat['chat_id']
chat = [i for i in chat_ids if i['alias_id'] == alias_id]
if not chat:
if not enable_otg:
return web.Response(status=403, text="403: Forbiden")
try:
chat_id = int(alias_id)
except:
return web.Response(status=403, text="403: Forbiden")
else:
chat = chat[0]
chat_id = chat['chat_id']
try:
message = await self.client.get_messages(entity=chat_id, ids=file_id)
except:
log.debug(f"Error in getting message {file_id} in {chat_id}", exc_info=True)
message = None
if not message or not message.file:
log.debug(f"no result for {file_id} in {chat_id}")
return web.Response(status=410, text="410: Gone. Access to the target resource is no longer available!")
media = message.media
size = message.file.size
file_name = get_file_name(message)
mime_type = message.file.mime_type
try:
offset = req.http_range.start or 0
limit = req.http_range.stop or size
@ -397,13 +461,13 @@ class Views:
"Content-Range": f"bytes */{size}"
}
)
if not head:
body = self.client.download(media, size, offset, limit)
log.info(f"Serving file in {message.id} (chat {chat_id}) ; Range: {offset} - {limit}")
else:
body = None
headers = {
"Content-Type": mime_type,
"Content-Range": f"bytes {offset}-{limit}/{size}",