Skip to content
Snippets Groups Projects
Commit fe3dca8e authored by Johann Jacobsohn's avatar Johann Jacobsohn
Browse files

introduce tinydb to cache data

parent 62cf7596
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
papersurfer/mattermost.py
+ 36
− 33
View file @ fe3dca8e
......@@ -9,16 +9,22 @@ from .doi import Doi
class Mattermost:
"""Provide a simplified interaction w/ mattermost api."""
def __init__(self, url, channelname, username, password):
self.msgs = []
self.mattermost = mattermostdriver.Driver({
self.posts = []
self._mattermost = mattermostdriver.Driver({
'url': url,
'login_id': username,
'password': password,
'port': 443
})
self._loggedin = False
self._reporters = {}
self._channelname = channelname
self._channel = None
def _login(self):
try:
self.mattermost.login()
self._mattermost.login()
except (mattermostdriver.exceptions.NoAccessTokenProvided,
requests.exceptions.InvalidURL,
requests.exceptions.HTTPError):
......@@ -26,16 +32,17 @@ class Mattermost:
raise ConfigError
try:
self.channel = self.get_channel(channelname)
self._channel = self._get_channel(self._channelname)
except ConfigError:
print("Couldn't find Mattermost channel.")
raise ConfigError
self.reporters = {}
def get_channel(self, channelname):
self._loggedin = True
def _get_channel(self, channelname):
"""Try to find the paper channel by display name."""
teamapi = self.mattermost.teams
channelapi = self.mattermost.channels
teamapi = self._mattermost.teams
channelapi = self._mattermost.channels
teams = [team["id"] for team in teamapi.get_user_teams("me")]
channels = []
for team in teams:
......@@ -50,46 +57,42 @@ class Mattermost:
raise ConfigError
return channels[0]["id"]
def get_reporter(self, userid):
def _get_reporter(self, userid):
"""Load user from mattermost api and cache."""
userapi = self.mattermost.users
if userid not in self.reporters:
self.reporters[userid] = userapi.get_user(userid)["username"]
userapi = self._mattermost.users
if userid not in self._reporters:
self._reporters[userid] = userapi.get_user(userid)["username"]
return self.reporters[userid]
return self._reporters[userid]
def retrieve_all_messages(self):
"""Retrieve all messages from mattermost, unfiltered for papers."""
posts = self.mattermost.posts.get_posts_for_channel(self.channel)
def _retrieve_all_posts(self):
"""Retrieve all posts from mattermost, unfiltered for papers."""
posts = self._mattermost.posts.get_posts_for_channel(self._channel)
return [PostDTO(id=m['id'], message=m['message'],
reporter=self.get_reporter(m['user_id']),
reporter=self._get_reporter(m['user_id']),
doi=Doi().extract_doi(m['message']),)
for m in posts['posts'].values()]
def filter_incoming(self, posts):
"""Filter messages from mattermost to only papers."""
def _filter_incoming(self, posts):
"""Filter posts from mattermost to only papers."""
return [p for p in posts if "doi" in p.message]
def retrieve(self):
"""Retrieve papers from mattermost channel."""
msgs = self.retrieve_all_messages()
self.msgs = self.filter_incoming(msgs)
return self.msgs
if not self._loggedin:
self._login()
posts = self._retrieve_all_posts()
self.posts = self._filter_incoming(posts)
return self.posts
def check_doi_exits(self, doi):
"""Check for doi in current paper list."""
doi_needle = Doi().extract_doi(doi)
msg_found = [msg for msg in self.msgs
if Doi().extract_doi(msg.doi) == doi_needle]
return bool(msg_found)
def get_filtered(self, needle):
"""Filter posts by needle."""
return [m for m in self.msgs
if needle.lower() in m.message.lower()
or needle.lower() in m.reporter.lower()]
posts_found = [posts for posts in self.posts
if Doi().extract_doi(posts.doi) == doi_needle]
return bool(posts_found)
def post(self, message):
"""Post message to thread."""
self.mattermost.posts.create_post({"channel_id": self.channel,
self._mattermost.posts.create_post({"channel_id": self._channel,
"message": message})
papersurfer/papersurfer.py
+ 95
− 13
View file @ fe3dca8e
......@@ -17,14 +17,91 @@ import sys
import re
import urwid
import configargparse
from tinydb import TinyDB, Query
from .exceptions import ConfigError
from .ui_elements import PrettyButton
from .mattermost import Mattermost
from .doi import Doi
from .bibtex import Bibtex
from .dtos import PostDTO
class Papersurfer:
"""Organize and cache paper/post data.
This handles interaction with mattermost, doi and a local database.
Papers and posts are similar but distinct concepts. A post contains
information on a single mattermost entry, containing a paper reference.
A paper contains information on a single scientific paper and a reference
back to the mattermost post.
"""
def __init__(self, url, channelname, username, password):
self._filters = {
"needle": "",
"fromdate": None,
"untildate": None
}
self.mattermost = Mattermost(url, channelname, username, password)
self.db_path = "."
self.isLoggedIn = False
self.db_posts = None
self.db_papers = None
self.db_file_posts = "papersurfer_posts_db.json"
self.db_file_papers = "papersurfer_papers_db.json"
def load(self):
"""Load data from mattermost and save to storage."""
self._connect_db()
posts = self.mattermost.retrieve()
self._update_db(posts=posts)
def _connect_db(self):
"""Establish db connection. Noop if already connected."""
if not self.db_posts:
self.db_posts = TinyDB(f"{self.db_path}/{self.db_file_posts}")
if not self.db_papers:
self.db_papers = TinyDB(f"{self.db_path}/{self.db_file_papers}")
def _update_db(self, posts=[], papers=[]):
""""Merge new data into database."""
self._upsert_multiple(posts, self.db_posts)
self._upsert_multiple(papers, self.db_papers)
def _upsert_multiple(self, records, db):
""""Update record in db unless it exits, then insert.
Would be trivial if we could just change the unique id in tinydb to the
doi property, but we can't.
"""
for record in records:
q = Query()
db.upsert(record.__dict__, q.doi == record.doi)
def get_posts(self):
"""Get all posts in storage."""
self._connect_db()
return [PostDTO(p["id"], p["message"], p["reporter"], p["doi"])
for p in self.db_posts.all()]
def get_posts_filtered(self, needle=None):
"""Return a list of papers, filtered by filter."""
self._filters['needle'] = needle = (needle
if needle
else self._filters['needle'])
return [m for m in self.get_posts()
if needle.lower() in m.message.lower()
or needle.lower() in m.reporter.lower()]
def get_papers(self):
"""Get all papers in storage."""
return self.db_papers.all()
class PapersurferUi:
"""Provide UI and interface with mattermost class."""
_palette = [
......@@ -39,9 +116,9 @@ class Papersurfer:
]
def __init__(self, url, channel, username, password):
self.papersurfer = Papersurfer(url, channel, username, password)
self._screen = urwid.raw_display.Screen()
self.size = self._screen.get_cols_rows()
self.filter = ""
ask = urwid.Edit(('I say', u"Filter?\n"))
exitbutton = PrettyButton(u'Exit', on_press=self.on_exit_clicked)
......@@ -51,8 +128,6 @@ class Papersurfer:
on_press=self.open_submit_paper)
div = urwid.Divider(u'-')
self.mtm = Mattermost(url, channel, username, password)
body = [urwid.Text("")]
self.listcontent = urwid.SimpleFocusListWalker(body)
......@@ -84,6 +159,7 @@ class Papersurfer:
self.mainloop = urwid.MainLoop(self._over, self._palette,
unhandled_input=self.h_unhandled_input)
self.mainloop.set_alarm_in(.1, self.load_list)
self.mainloop.set_alarm_in(.2, self.update_data)
self.mainloop.run()
def h_unhandled_input(self, key):
......@@ -93,11 +169,18 @@ class Papersurfer:
def load_list(self, _loop, _data):
"""Load and display paper list."""
body = [self.list_item(paper) for paper in self.mtm.retrieve()]
body = [self.list_item(post) for post in self.papersurfer.get_posts()]
if len(body) == 0:
return
self.listcontent.clear()
self.listcontent.extend(body)
self.mainloop.widget = self.top
def update_data(self, _loop, _data):
"""Load and display paper list."""
self.papersurfer.load()
self.mainloop.set_alarm_in(.1, self.load_list)
def loading_indicator(self):
"""Create loading indicator dialog."""
body_text = urwid.Text("Loading...", align='center')
......@@ -179,20 +262,19 @@ class Papersurfer:
def onchange(self, _, needle):
"""Handle filter change."""
self.filter = needle
self.listcontent.clear()
self.listcontent.extend([self.list_item(paper, needle)
for paper in self.mtm.get_filtered(needle)])
self.listcontent.extend([
self.list_item(paper, needle)
for paper in self.papersurfer.get_posts_filtered(needle)])
def running_export(self, state):
"""Set exporting state."""
btn = self.exportbutton
label = btn.get_label()
label = self.exportbutton.get_label()
running_indicator = " (running...)"
if state:
btn.set_label(label + running_indicator)
self.exportbutton.set_label(label + running_indicator)
else:
btn.set_label(label.replace(running_indicator, ""))
self.exportbutton.set_label(label.replace(running_indicator, ""))
self.updscrn()
def on_exit_clicked(self, button):
......@@ -207,7 +289,7 @@ class Papersurfer:
def export_to_bibtex(self):
"""Export current filtered list to bibtex file."""
papers = self.mtm.get_filtered(self.filter)
papers = self.papersurfer.get_posts_filtered()
dois = [paper.doi for paper in papers]
string = Bibtex().bib_from_dois(dois)
with open("export.bib", 'w') as file:
......@@ -487,7 +569,7 @@ def main():
if opt.dump_bibtex:
just_bibtex(opt.url, opt.channel, opt.username, opt.password)
else:
Papersurfer(opt.url, opt.channel, opt.username, opt.password)
PapersurferUi(opt.url, opt.channel, opt.username, opt.password)
if __name__ == "__main__":
......
setup.py
+ 3
− 2
View file @ fe3dca8e
......@@ -15,7 +15,6 @@ setup(
description="",
long_description=README,
long_description_content_type="text/markdown",
url="",
author="Johann Jacobsohn",
author_email="johann.jacobsohn@uni-hamburg.de",
license="MIT",
......@@ -23,10 +22,12 @@ setup(
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.7",
'Development Status :: 1 - Planning',
],
packages=["papersurfer"],
include_package_data=True,
install_requires=["requests", "mattermostdriver", "urwid", "configargparse"],
install_requires=["requests", "mattermostdriver", "urwid",
"configargparse", "tinydb"],
entry_points={
"console_scripts": [
"papersurfer=papersurfer.papersurfer:main",
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment