diff --git a/papersurfer/mattermost.py b/papersurfer/mattermost.py
index a549ce6687d18e8e51b882c3a83887f7ab0cd71d..140ffc0f67f011bff75cf5d923d4830d9697f82f 100644
--- a/papersurfer/mattermost.py
+++ b/papersurfer/mattermost.py
@@ -9,16 +9,22 @@ from .doi import Doi
class Mattermost:
"""Provide a simplified interaction w/ mattermost api."""
def __init__(self, url, channelname, username, password):
- self.msgs = []
- self.mattermost = mattermostdriver.Driver({
+ self.posts = []
+ self._mattermost = mattermostdriver.Driver({
'url': url,
'login_id': username,
'password': password,
'port': 443
})
+ self._loggedin = False
+ self._reporters = {}
+ self._channelname = channelname
+ self._channel = None
+
+ def _login(self):
try:
- self.mattermost.login()
+ self._mattermost.login()
except (mattermostdriver.exceptions.NoAccessTokenProvided,
requests.exceptions.InvalidURL,
requests.exceptions.HTTPError):
@@ -26,16 +32,17 @@ class Mattermost:
raise ConfigError
try:
- self.channel = self.get_channel(channelname)
+ self._channel = self._get_channel(self._channelname)
except ConfigError:
print("Couldn't find Mattermost channel.")
raise ConfigError
- self.reporters = {}
- def get_channel(self, channelname):
+ self._loggedin = True
+
+ def _get_channel(self, channelname):
"""Try to find the paper channel by display name."""
- teamapi = self.mattermost.teams
- channelapi = self.mattermost.channels
+ teamapi = self._mattermost.teams
+ channelapi = self._mattermost.channels
teams = [team["id"] for team in teamapi.get_user_teams("me")]
channels = []
for team in teams:
@@ -50,46 +57,42 @@ class Mattermost:
raise ConfigError
return channels[0]["id"]
- def get_reporter(self, userid):
+ def _get_reporter(self, userid):
"""Load user from mattermost api and cache."""
- userapi = self.mattermost.users
- if userid not in self.reporters:
- self.reporters[userid] = userapi.get_user(userid)["username"]
+ userapi = self._mattermost.users
+ if userid not in self._reporters:
+ self._reporters[userid] = userapi.get_user(userid)["username"]
- return self.reporters[userid]
+ return self._reporters[userid]
- def retrieve_all_messages(self):
- """Retrieve all messages from mattermost, unfiltered for papers."""
- posts = self.mattermost.posts.get_posts_for_channel(self.channel)
+ def _retrieve_all_posts(self):
+ """Retrieve all posts from mattermost, unfiltered for papers."""
+ posts = self._mattermost.posts.get_posts_for_channel(self._channel)
return [PostDTO(id=m['id'], message=m['message'],
- reporter=self.get_reporter(m['user_id']),
+ reporter=self._get_reporter(m['user_id']),
doi=Doi().extract_doi(m['message']),)
for m in posts['posts'].values()]
- def filter_incoming(self, posts):
- """Filter messages from mattermost to only papers."""
+ def _filter_incoming(self, posts):
+ """Filter posts from mattermost to only papers."""
return [p for p in posts if "doi" in p.message]
def retrieve(self):
"""Retrieve papers from mattermost channel."""
- msgs = self.retrieve_all_messages()
- self.msgs = self.filter_incoming(msgs)
- return self.msgs
+ if not self._loggedin:
+ self._login()
+ posts = self._retrieve_all_posts()
+ self.posts = self._filter_incoming(posts)
+ return self.posts
def check_doi_exits(self, doi):
"""Check for doi in current paper list."""
doi_needle = Doi().extract_doi(doi)
- msg_found = [msg for msg in self.msgs
- if Doi().extract_doi(msg.doi) == doi_needle]
- return bool(msg_found)
-
- def get_filtered(self, needle):
- """Filter posts by needle."""
- return [m for m in self.msgs
- if needle.lower() in m.message.lower()
- or needle.lower() in m.reporter.lower()]
+ posts_found = [posts for posts in self.posts
+ if Doi().extract_doi(posts.doi) == doi_needle]
+ return bool(posts_found)
def post(self, message):
"""Post message to thread."""
- self.mattermost.posts.create_post({"channel_id": self.channel,
- "message": message})
+ self._mattermost.posts.create_post({"channel_id": self._channel,
+ "message": message})
diff --git a/papersurfer/papersurfer.py b/papersurfer/papersurfer.py
index bb809b9692fb30353787b5593de9196561b60c5f..48a97c00d5e1a05f8a87903f22ec4e099c3b84a8 100644
--- a/papersurfer/papersurfer.py
+++ b/papersurfer/papersurfer.py
@@ -17,14 +17,91 @@ import sys
import re
import urwid
import configargparse
+from tinydb import TinyDB, Query
from .exceptions import ConfigError
from .ui_elements import PrettyButton
from .mattermost import Mattermost
from .doi import Doi
from .bibtex import Bibtex
+from .dtos import PostDTO
class Papersurfer:
+ """Organize and cache paper/post data.
+
+ This handles interaction with mattermost, doi and a local database.
+
+ Papers and posts are similar but distinct concepts. A post contains
+ information on a single mattermost entry, containing a paper reference.
+ A paper contains information on a single scientific paper and a reference
+ back to the mattermost post.
+ """
+ def __init__(self, url, channelname, username, password):
+ self._filters = {
+ "needle": "",
+ "fromdate": None,
+ "untildate": None
+ }
+
+ self.mattermost = Mattermost(url, channelname, username, password)
+
+ self.db_path = "."
+
+ self.isLoggedIn = False
+ self.db_posts = None
+ self.db_papers = None
+ self.db_file_posts = "papersurfer_posts_db.json"
+ self.db_file_papers = "papersurfer_papers_db.json"
+
+ def load(self):
+ """Load data from mattermost and save to storage."""
+ self._connect_db()
+ posts = self.mattermost.retrieve()
+ self._update_db(posts=posts)
+
+ def _connect_db(self):
+ """Establish db connection. Noop if already connected."""
+ if not self.db_posts:
+ self.db_posts = TinyDB(f"{self.db_path}/{self.db_file_posts}")
+ if not self.db_papers:
+ self.db_papers = TinyDB(f"{self.db_path}/{self.db_file_papers}")
+
+ def _update_db(self, posts=[], papers=[]):
+ """"Merge new data into database."""
+ self._upsert_multiple(posts, self.db_posts)
+ self._upsert_multiple(papers, self.db_papers)
+
+ def _upsert_multiple(self, records, db):
+ """"Update record in db unless it exits, then insert.
+
+ Would be trivial if we could just change the unique id in tinydb to the
+ doi property, but we can't.
+ """
+ for record in records:
+ q = Query()
+ db.upsert(record.__dict__, q.doi == record.doi)
+
+ def get_posts(self):
+ """Get all posts in storage."""
+ self._connect_db()
+ return [PostDTO(p["id"], p["message"], p["reporter"], p["doi"])
+ for p in self.db_posts.all()]
+
+ def get_posts_filtered(self, needle=None):
+ """Return a list of papers, filtered by filter."""
+ self._filters['needle'] = needle = (needle
+ if needle
+ else self._filters['needle'])
+ return [m for m in self.get_posts()
+ if needle.lower() in m.message.lower()
+ or needle.lower() in m.reporter.lower()]
+
+ def get_papers(self):
+ """Get all papers in storage."""
+ return self.db_papers.all()
+
+
+class PapersurferUi:
"""Provide UI and interface with mattermost class."""
_palette = [
@@ -39,9 +116,9 @@ class Papersurfer:
]
def __init__(self, url, channel, username, password):
+ self.papersurfer = Papersurfer(url, channel, username, password)
self._screen = urwid.raw_display.Screen()
self.size = self._screen.get_cols_rows()
- self.filter = ""
ask = urwid.Edit(('I say', u"Filter?\n"))
exitbutton = PrettyButton(u'Exit', on_press=self.on_exit_clicked)
@@ -51,8 +128,6 @@ class Papersurfer:
on_press=self.open_submit_paper)
div = urwid.Divider(u'-')
- self.mtm = Mattermost(url, channel, username, password)
-
body = [urwid.Text("")]
self.listcontent = urwid.SimpleFocusListWalker(body)
@@ -84,6 +159,7 @@ class Papersurfer:
self.mainloop = urwid.MainLoop(self._over, self._palette,
unhandled_input=self.h_unhandled_input)
self.mainloop.set_alarm_in(.1, self.load_list)
+ self.mainloop.set_alarm_in(.2, self.update_data)
self.mainloop.run()
def h_unhandled_input(self, key):
@@ -93,11 +169,18 @@ class Papersurfer:
def load_list(self, _loop, _data):
"""Load and display paper list."""
- body = [self.list_item(paper) for paper in self.mtm.retrieve()]
+ body = [self.list_item(post) for post in self.papersurfer.get_posts()]
+ if len(body) == 0:
+ return
self.listcontent.clear()
self.listcontent.extend(body)
self.mainloop.widget = self.top
+ def update_data(self, _loop, _data):
+ """Load and display paper list."""
+ self.papersurfer.load()
+ self.mainloop.set_alarm_in(.1, self.load_list)
+
def loading_indicator(self):
"""Create loading indicator dialog."""
body_text = urwid.Text("Loading...", align='center')
@@ -179,20 +262,19 @@ class Papersurfer:
def onchange(self, _, needle):
"""Handle filter change."""
- self.filter = needle
self.listcontent.clear()
- self.listcontent.extend([self.list_item(paper, needle)
- for paper in self.mtm.get_filtered(needle)])
+ self.listcontent.extend([
+ self.list_item(paper, needle)
+ for paper in self.papersurfer.get_posts_filtered(needle)])
def running_export(self, state):
"""Set exporting state."""
- btn = self.exportbutton
- label = btn.get_label()
+ label = self.exportbutton.get_label()
running_indicator = " (running...)"
if state:
- btn.set_label(label + running_indicator)
+ self.exportbutton.set_label(label + running_indicator)
else:
- btn.set_label(label.replace(running_indicator, ""))
+ self.exportbutton.set_label(label.replace(running_indicator, ""))
self.updscrn()
def on_exit_clicked(self, button):
@@ -207,7 +289,7 @@ class Papersurfer:
def export_to_bibtex(self):
"""Export current filtered list to bibtex file."""
- papers = self.mtm.get_filtered(self.filter)
+ papers = self.papersurfer.get_posts_filtered()
dois = [paper.doi for paper in papers]
string = Bibtex().bib_from_dois(dois)
with open("export.bib", 'w') as file:
@@ -487,7 +569,7 @@ def main():
if opt.dump_bibtex:
just_bibtex(opt.url, opt.channel, opt.username, opt.password)
else:
- Papersurfer(opt.url, opt.channel, opt.username, opt.password)
+ PapersurferUi(opt.url, opt.channel, opt.username, opt.password)
if __name__ == "__main__":
diff --git a/setup.py b/setup.py
index 0ad45200f20ad2652ff97a61f320f0c748cda4cc..5080f19ce789288f1131a4111543eb4ac5d7b945 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,6 @@ setup(
description="",
long_description=README,
long_description_content_type="text/markdown",
- url="",
author="Johann Jacobsohn",
author_email="johann.jacobsohn@uni-hamburg.de",
license="MIT",
@@ -23,10 +22,12 @@ setup(
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.7",
+ 'Development Status :: 1 - Planning',
],
packages=["papersurfer"],
include_package_data=True,
- install_requires=["requests", "mattermostdriver", "urwid", "configargparse"],
+ install_requires=["requests", "mattermostdriver", "urwid",
+ "configargparse", "tinydb"],
entry_points={
"console_scripts": [
"papersurfer=papersurfer.papersurfer:main",