add multiple channel index support

This commit is contained in:
odysseusmax 2020-08-12 10:39:29 +00:00
parent 81679a37c0
commit da02bad347
9 changed files with 87 additions and 32 deletions

View File

@ -37,7 +37,7 @@ $ pip3 install -U -r requirements.txt
|------------- | -------------
| `API_ID` (required) | Telegram api_id obtained from https://my.telegram.org/apps.
| `API_HASH` (required) | Telegram api_hash obtained from https://my.telegram.org/apps.
| `CHAT_ID` (required) | Id of the telegram channel (or chat) to be indexed.
| `CHAT_ID` (required) | Id of the telegram channel (or chat) to be indexed. Separate id's with space if you want to index more than one channel.
| `SESSION_STRING` (required) | String obtained by running `$ python3 app/generate_session_string.py`. (Login with the telegram account which is a participant of the given channel (or chat).
| `PORT` (optional) | Port on which app should listen to, defaults to 8080.
| `HOST` (optional) | Host name on which app should listen to, defaults to 0.0.0.0.

View File

@ -13,7 +13,7 @@
"value":""
},
"CHAT_ID":{
"description":"Id of the telegram channel (or chat) to be indexed.",
"description":"Id of the telegram channel (or chat) to be indexed. Separate id's with space if you want to index more than one channel.",
"value":""
},
"SESSION_STRING":{

View File

@ -16,6 +16,7 @@ TEMPLATES_ROOT = pathlib.Path(__file__).parent / 'templates'
client = Client(session_string, api_id, api_hash)
logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
logging.getLogger("telethon").setLevel(logging.INFO if debug else logging.ERROR)
logging.getLogger("aiohttp").setLevel(logging.INFO if debug else logging.ERROR)
log = logging.getLogger("tg-index")

View File

@ -19,7 +19,8 @@ except (KeyError, ValueError):
sys.exit(1)
try:
chat_id = int(os.environ["CHAT_ID"])
chat_id_raw = os.environ["CHAT_ID"]
chat_ids = [int(chat_id) for chat_id in chat_id_raw.split(' ')]
except (KeyError, ValueError):
print("Please set the CHAT_ID environment variable correctly")
sys.exit(1)

View File

@ -1,15 +1,22 @@
from aiohttp import web
from config import chat_ids
def setup_routes(app, handler):
h = handler
app.add_routes(
[
web.get('/', h.index, name='index'),
web.get(r"/{id:\d+}/view", h.info, name='info'),
web.get(r"/{id:\d+}/download", h.download_get),
web.head(r"/{id:\d+}/download", h.download_head),
web.get(r"/{id:\d+}/thumbnail", h.thumbnail_get),
web.head(r"/{id:\d+}/thumbnail", h.thumbnail_head),
routes = [
web.get('/', h.home, name='home')
]
for chat_id in chat_ids:
p = f"/{chat_id}"
r = [
web.get(p, h.index),
web.get(p + r"/{id:\d+}/view", h.info),
web.get(p + r"/{id:\d+}/download", h.download_get),
web.head(p + r"/{id:\d+}/download", h.download_head),
web.get(p + r"/{id:\d+}/thumbnail", h.thumbnail_get),
web.head(p + r"/{id:\d+}/thumbnail", h.thumbnail_head),
]
)
routes += r
app.add_routes(routes)

12
app/templates/home.html Normal file
View File

@ -0,0 +1,12 @@
{% include 'header.html' %}
<h1 class=" my-2 text-2xl text-center font-bold text-green-400">
Available Sources
</h1>
<div class="mx-auto my-2 px-4 w-full">
{% for chat in chats %}
<li class="w-full border border-blue-300 rounded bg-blue-100 hover:bg-blue-200 p-4 my-3 shadow-md"> <a href="/{{chat.id}}" > {{chat.name}}</a> </li>
{% endfor %}
</div>
{% include 'footer.html' %}

View File

@ -1,8 +1,18 @@
{% include 'header.html' %}
<form class="p-4" action="/">
<input class="border rounded p-2 border-indigo-500 placeholder-indigo-500 text-indigo-500" type="text" name="search" value="{% if search %}{{search}}{% endif %}" placeholder="search...">
<button class="border rounded p-2 border-indigo-500 bg-indigo-500 text-white hover:text-indigo-500 hover:bg-white" type="submit">Search</button>
</form>
<div class="block md:flex justify-between px-4 text-center">
<div class="my-2 text-2xl md:text-right font-bold text-blue-500">
<h1> {{name}} </h1>
</div>
<div class="my-2">
<form class="">
<input class="border rounded p-2 border-indigo-500 placeholder-indigo-500 text-indigo-500" type="text" name="search" value="{% if search %}{{search}}{% endif %}" placeholder="search...">
<button class="border rounded p-2 border-indigo-500 bg-indigo-500 text-white hover:text-indigo-500 hover:bg-white" type="submit">Search</button>
</form>
</div>
</div>
<div class="mx-auto my-2 w-full overflow-x-auto px-4 text-xs">
@ -21,7 +31,7 @@
<th class="my-2 p-2">{{item.file_id}}</th>
<th class="my-2 p-2">{{item.media}}</th>
<th class="my-2 p-2">{{item.mime_type}}</th>
<th class="my-2 p-2 rounded hover:text-blue-600"><a href="/{{item.file_id}}/view">{{item.insight}}</a></th>
<th class="my-2 p-2 rounded hover:text-blue-600"><a href="{{item.url}}">{{item.insight}}</a></th>
<th class="my-2 p-2">{{item.date}}</th>
<th class="my-2 p-2">{{item.size}}</th>
</tr>

View File

@ -9,13 +9,13 @@
<div class="mx-auto w-full md:w-3/4 lg:w-2/5 p-2">
{% if media.image %}
<img class="mx-auto w-2/3 md:w-3/5 lg:w-2/5 rounded" src="/{{id}}/thumbnail" alt="{{name}}">
<img class="mx-auto w-2/3 md:w-3/5 lg:w-2/5 rounded" src="../{{id}}/thumbnail" alt="{{name}}">
{% elif media.video %}
<video id="my-video-player" class="mx-auto rounded" controls poster="/{{id}}/thumbnail">
<video id="my-video-player" class="mx-auto rounded" controls poster="../{{id}}/thumbnail">
<source src="/{{id}}/download" type="{{ media.type }}" />
</video>
{% elif media.audio %}
<audio class="mx-auto" controls autoplay muted src="/{{id}}/download" type="{{ media.type }}"></audio>
<audio class="mx-auto" controls autoplay muted src="../{{id}}/download" type="{{ media.type }}"></audio>
{% endif %}
{% if caption %}
@ -30,7 +30,7 @@
<div class="text-center text-white text-sm md:text-base xl:text-2xl">
<a class="rounded p-3 bg-indigo-500 hover:bg-indigo-700 shadow-lg" href="/{{id}}/download">Download Now! ({{ size }})</a>
<a class="rounded p-3 bg-indigo-500 hover:bg-indigo-700 shadow-lg" href="../{{id}}/download">Download Now! ({{ size }})</a>
</div>
{% else %}
@ -88,4 +88,4 @@
);
</script>
{% include 'footer.html' %}
{% include 'footer.html' %}

View File

@ -7,7 +7,7 @@ from telethon.tl import types
from telethon.tl.custom import Message
from .util import get_file_name, get_human_size
from .config import chat_id
from .config import chat_ids
log = logging.getLogger(__name__)
@ -18,9 +18,25 @@ class Views:
def __init__(self, client):
self.client = client
@aiohttp_jinja2.template('home.html')
async def home(self, req):
if len(chat_ids) == 1:
raise web.HTTPFound(f"{chat_ids[0]}")
chats = []
for chat_id in chat_ids:
chat = await self.client.get_entity(chat_id)
chats.append({
'id': chat_id,
'name': chat.title
})
return {'chats':chats}
@aiohttp_jinja2.template('index.html')
async def index(self, req):
chat_id = int(req.rel_url.path.split('/')[1])
chat = await self.client.get_entity(chat_id)
log_msg = ''
try:
offset_val = int(req.query.get('page', '1'))
@ -34,14 +50,19 @@ class Views:
log_msg += f"search query: {search_query} | "
offset_val = 0 if offset_val <=1 else offset_val-1
try:
kwargs = {
'entity': chat_id,
'limit': 20,
'add_offset': 20*offset_val
}
if search_query:
messages = (await self.client.get_messages(chat_id, search=search_query, limit=20, add_offset=20*offset_val)) or []
else:
messages = (await self.client.get_messages(chat_id, limit=20, add_offset=20*offset_val)) or []
kwargs.update({'search': search_query})
messages = (await self.client.get_messages(**kwargs)) or []
except:
log.debug("failed to get messages", exc_info=True)
messages = []
log_msg += f"found {len(messages)} results"
log_msg += f"found {len(messages)} results | "
log.debug(log_msg)
results = []
for m in messages:
@ -52,7 +73,8 @@ class Views:
mime_type=m.file.mime_type,
insight = get_file_name(m)[:55],
date = m.date.isoformat(),
size=get_human_size(m.file.size)
size=get_human_size(m.file.size),
url=req.rel_url.with_path(f"/{chat_id}/{m.id}/view")
)
elif m.message:
entry = dict(
@ -61,7 +83,8 @@ class Views:
mime_type='text/plain',
insight = m.raw_text[:55],
date = m.date.isoformat(),
size=get_human_size(len(m.raw_text))
size=get_human_size(len(m.raw_text)),
url=req.rel_url.with_path(f"/{chat_id}/{m.id}/view")
)
results.append(entry)
prev_page = False
@ -90,12 +113,14 @@ class Views:
'cur_page' : offset_val+1,
'next_page': next_page,
'search': search_query,
'name' : chat.title
}
@aiohttp_jinja2.template('info.html')
async def info(self, req):
file_id = int(req.match_info["id"])
chat_id = int(req.rel_url.path.split('/')[1])
message = await self.client.get_messages(entity=chat_id, ids=file_id)
if not message or not isinstance(message, Message):
log.debug(f"no valid entry for {file_id} in {chat_id}")
@ -168,7 +193,7 @@ class Views:
async def handle_request(self, req, head=False, thumb=False):
file_id = int(req.match_info["id"])
chat_id = int(req.rel_url.path.split('/')[1])
message = await self.client.get_messages(entity=chat_id, ids=file_id)
if not message or not message.file:
log.info(f"no result for {file_id} in {chat_id}")
@ -210,11 +235,10 @@ class Views:
}
)
body = None
if not head:
body = self.client.download(media, size, offset, limit)
log.info(f"Serving file {message.id} in {chat_id} ; Range: {offset} - {limit}")
else:
body = None
headers = {
"Content-Type": mime_type,