diff --git a/app.py b/app.py
index 9b7e7d26fe8569e65558c9317750fa0bfb399cdc..45838da38951f9f96b7b2f5f7bea860eb0a2d003 100644
--- a/app.py
+++ b/app.py
@@ -18,7 +18,7 @@ from werkzeug.utils import redirect
from wtforms import validators
import flask_admin as admin
-from flask_admin.base import MenuLink
+from flask_admin.base import MenuLink, AdminIndexView
from flask_admin.contrib import sqla
from flask_admin.contrib.sqla import filters
from flask_admin.contrib.sqla.form import InlineModelConverter
@@ -171,9 +171,9 @@ class Reference(db.Model):
# Flask views
-@app.route('/')
-def index():
- return redirect(url_for("admin.index"))
+#@app.route('/')
+#def index():
+# return redirect(url_for("admin.index"))
class AdvancedSoftwareView(sqla.ModelView):
@@ -252,11 +252,12 @@ class AdvancedSoftwareView(sqla.ModelView):
result = db.session.query(hierarchy.c.level, hierarchy.c.id, hierarchy.c.parent_id, hierarchy.c.name, hierarchy.c.description)\
.select_entity_from(hierarchy).order_by(hierarchy.c.id)\
.all()
+ references = db.session.query(Reference).order_by(Reference.name).all()
# Generate sub pages
with open('templates/export/MethodsList.jinja2', "r", encoding="utf-8") as file_:
template = Template(file_.read())
- template.stream(methods=result).dump('../digitale-Methoden-wiki/MethodsList.asciidoc', encoding='utf-8')
+ template.stream(methods=result, references=references).dump('../digitale-Methoden-wiki/MethodsList.asciidoc', encoding='utf-8')
flash("Files generated!")
except Exception as ex:
if not self.handle_view_exception(ex):
@@ -264,18 +265,22 @@ class AdvancedSoftwareView(sqla.ModelView):
flash("Not done")
# Create admin
-admin = admin.Admin(app, name='digital methods:software-tools', template_mode='bootstrap3')
+admin = admin.Admin(app, name='digital methods:software-tools', template_mode='bootstrap3', index_view=AdminIndexView(
+ name='home',
+ template='admin/index.html',
+ url='/'
+ ))
# Add views
-admin.add_view(AdvancedSoftwareView(Software, db.session, name="Software-Tools"))
+admin.add_view(AdvancedSoftwareView(Software, db.session, name="software-tools"))
admin.add_view(sqla.ModelView(Method, db.session, name="methods"))
-admin.add_view(sqla.ModelView(Feature, db.session, category="Misc"))
-admin.add_view(sqla.ModelView(License, db.session, category="Misc"))
-admin.add_view(sqla.ModelView(Link, db.session, category="Misc"))
-admin.add_view(sqla.ModelView(SoftwareCategory, db.session, category="Misc"))
-admin.add_sub_category(name="Other collections", parent_name="Misc")
-admin.add_link(MenuLink(name="CRAN-R", url='https://cran.r-project.org/web/views/', category='Other collections', target="_blank"))
-admin.add_link(MenuLink(name="ROpenSci", url='https://ropensci.org/packages/', category='Other collections', target="_blank"))
+admin.add_view(sqla.ModelView(Feature, db.session, category="miscellaneous"))
+admin.add_view(sqla.ModelView(License, db.session, category="miscellaneous"))
+admin.add_view(sqla.ModelView(Link, db.session, category="miscellaneous"))
+admin.add_view(sqla.ModelView(SoftwareCategory, db.session, category="miscellaneous"))
+admin.add_sub_category(name="other collections", parent_name="miscellaneous")
+admin.add_link(MenuLink(name="CRAN-R", url='https://cran.r-project.org/web/views/', category='other collections', target="_blank"))
+admin.add_link(MenuLink(name="ROpenSci", url='https://ropensci.org/packages/', category='other collections', target="_blank"))
@@ -289,165 +294,235 @@ def build_sample_db():
method = Method(id=1,
name="digital methods",
- description="",
+ description="""<<Rogers2013>> distinguishes between digitalized/virtual and digital methods. The former methods import standard methods from the social sciences and humanities into the emerging medium. The latter are completly new methods which emerge following the new structures and their properties. +
+In this project a more inclusive conception of digital methods is assumed: the potential use of digital technology during the research.""",
parent=None)
db.session.add(method)
method1 = Method(id=2,
name="data mining",
- description="",
+ description="""Refers to the complete process of 'knowledge mining from data'.<<Han_etal2012>> Can be applied on various data types and consists of different steps and paradigms.""",
parent=method)
db.session.add(method1)
method2 = Method(id=3,
name="automated data collection",
- description="",
+ description="""In principal there are multiple possible data sources in a data mining process. A basic distinction in relevance to automated data collection can be drawn between connected devices(internet, intranets) or unconnected devices(sensors, etc.). +
+Furthermore the server-client-model is the established communication paradigms for connected devices. In order to obtain data either from server or client there exists three different interfaces: log files, apis and user interfaces which constitute the available procedures <<Jünger2018>>.""",
parent=method1)
db.session.add(method2)
method3 = Method(id=4,
name="collect log-data",
- description="",
+ description="Collect log data which occur during providing the (web-)service or the information processing.",
parent=method2)
db.session.add(method3)
method3 = Method(id=5,
name="parsing from api",
- description="",
+ description="Parse structured data from via a documented REST-API.",
parent=method2)
db.session.add(method3)
method3 = Method(id=6,
name="scraping",
- description="",
+ description="Automatically parse unstructured or semi-structured data from a normal website (⇒ web-scraping) or service.",
parent=method2)
db.session.add(method3)
method4 = Method(id=7,
name="scraping (static content)",
- description="",
+ description="Automatically parse data from static HTML websites.",
parent=method3)
db.session.add(method4)
method4 = Method(id=8,
name="scraping (dynamic content)",
- description="",
+ description="Automatically parse dynamic content (HTML5/Javascript,) ⇒ sometimes requires mimicking user-interaction.",
parent=method3)
db.session.add(method4)
method3 = Method(id=9,
name="crawling",
- description="",
+ description="Collect websites with an initial set of webpages by following contained links <<Ignatow_etal2017>>.",
parent=method2)
db.session.add(method3)
method2 = Method(id=10,
name="data wrangling",
- description="",
+ description="Translate data into suited formats for automatic analysis. Examples: PDFs ⇒ Text . For a practical framework refer also <<Wickham_etal2017>>.",
parent=method1)
db.session.add(method2)
method3 = Method(id=11,
name="regular expressions",
- description="",
+ description="Complex string manipulations by searching and replacing specific patterns.",
parent=method2)
db.session.add(method3)
method3 = Method(id=12,
- name="format conversions",
- description="",
+ name="data-format conversions",
+ description="Transfer between different formats in order to unify and handle vacancies.",
parent=method2)
db.session.add(method3)
- method3 = Method(id=13,
- name="language preprocessing",
- description="",
+ method2 = Method(id=13,
+ name="text preprocessing",
+ description="Some text preprocessing tasks in natuaral language processing.",
+ parent=method1)
+ db.session.add(method2)
+ method3 = Method(id=14,
+ name="tokenization",
+ description="Identify words in character input sequence.",
parent=method2)
db.session.add(method3)
- method2 = Method(id=14, name="information extraction",
- description="finding factual information in free text",
+ method3 = Method(id=15,
+ name="stop-word removal",
+ description="Removing high-frequency words like pronoums, determiners or prepositions.",
+ parent=method2)
+ db.session.add(method3)
+ method3 = Method(id=16,
+ name="stemming",
+ description="Identify common stems on a syntactical level.",
+ parent=method2)
+ db.session.add(method3)
+ method3 = Method(id=17,
+ name="word/sentence segmentation",
+ description="Separate a chunk of continuous text into separate words/sentences.",
+ parent=method2)
+ db.session.add(method3)
+ method3 = Method(id=18,
+ name="part-of-speech(POS)-tagging",
+ description="Identify the part of speech for words.",
+ parent=method2)
+ db.session.add(method3)
+ method3 = Method(id=19,
+ name="dependency parsing",
+ description="Create syntactic trees from input text using mostly unsupervised learning on manually annotated treebanks (<<Ignatow_etal2017>>,61).",
+ parent=method2)
+ db.session.add(method3)
+ method4 = Method(id=20,
+ name="syntactic parsing",
+ description="Create syntactic trees from input text using mostly unsupervised learning on manually annotated treebanks (<<Ignatow_etal2017>>,61).",
+ parent=method3)
+ db.session.add(method4)
+ method3 = Method(id=21,
+ name="word-sense disambiguation",
+ description="Recognizing context-sensetive meaning of words.",
+ parent=method2)
+ db.session.add(method3)
+ method2 = Method(id=22, name="information extraction",
+ description="Finding factual information(e.g. places in free text",
parent=method1)
- db.session.add(method2)
- method2 = Method(id=15,name="information retrieval",
+ db.session.add(method3)
+ method3 = Method(id=23,
+ name="(named-)entity-recognition/resolution/extraction",
+ description="Identify instances of specific (pre-)defined types(e.g place, name or color) in text.",
+ parent=method2)
+ db.session.add(method3)
+ method3 = Method(id=24,
+ name="relation extraction",
+ description="",
+ parent=method3)
+ db.session.add(method3)
+ method2 = Method(id=25,name="information retrieval",
description="",
parent=method1)
db.session.add(method2)
- method2 = Method(id=16,name="statistical analysis",
+ method2 = Method(id=26,name="statistical analysis",
description="",
parent=method1)
db.session.add(method2)
- method3 = Method(id=17,name="frequency analysis",
- description="",
+ method3 = Method(id=27,name="frequency analysis",
+ description="Descriptiv statistical analysis by using text fragments",
parent=method2)
db.session.add(method3)
- method3 = Method(id=18,name="classification/machine learning",
+ method3 = Method(id=28,name="classification/machine learning",
description="",
parent=method2)
db.session.add(method3)
- method4 = Method(id=19,name="supervised classification",
+ method4 = Method(id=29,name="supervised classification",
description="",
parent=method3)
db.session.add(method4)
- method4 = Method(id=20,name="topic models",
+ method4 = Method(id=30,name="topic models",
description="",
parent=method3)
db.session.add(method4)
- method4 = Method(id=21,name="sentiment analysis",
- description="",
+ method4 = Method(id=31,name="sentiment analysis",
+ description="'Subjectivity and sentiment analysis focuses on the automatic identification of private states, such as opinions, emotions, sentiments, evaluations, beliefs, and speculations in natural language. While subjectivity classification labels text as either subjective or objective, sentiment classification adds an additional level of granularity, by further classifying subjective text as either positive, negative, or neutral.' (<<Ignatow_etal2017>> pp. 148)",
parent=method3)
db.session.add(method4)
- method4 = Method(id=22,name="automated narrative, argumentative structures, irony, metaphor detection/extraction",
+ method4 = Method(id=32,name="automated narrative, argumentative structures, irony, metaphor detection/extraction",
description="",
parent=method3)
db.session.add(method4)
- method3 = Method(id=23,name="network analysis",
+ method3 = Method(id=33,name="network analysis",
description="",
parent=method2)
db.session.add(method3)
- method4 = Method(id=24, name="knowledge graph construction",
+ method4 = Method(id=34, name="knowledge graph construction",
description="finding factual information in free text",
parent=method3)
db.session.add(method4)
- method2 = Method(id=25,name="data visualization",
+ method2 = Method(id=35,name="data visualization",
description="",
parent=method1)
db.session.add(method2)
- method3 = Method(id=26,name="dynamic visualizations",
+ method3 = Method(id=36,name="dynamic visualizations",
description="",
parent=method2)
db.session.add(method3)
- method1 = Method(id=27,name="science practice",
+ method1 = Method(id=37,name="science practice",
description="",
parent=method)
db.session.add(method1)
- method2 = Method(id=28,name="digital research design",
+ method2 = Method(id=38,name="digital research design",
description="",
parent=method1)
db.session.add(method2)
- method2 = Method(id=29,name="collaborative work",
+ method2 = Method(id=39,name="collaborative work",
description="",
parent=method1)
db.session.add(method2)
- method2 = Method(id=30,name="digital communication",
+ method2 = Method(id=40,name="digital communication",
description="",
parent=method1)
db.session.add(method2)
- method1 = Method(id=31,name="statistical modeling",
+ method1 = Method(id=41,name="statistical modeling",
description="",
parent=method)
db.session.add(method1)
- method2 = Method(id=32,name="regression analysis",
+ method2 = Method(id=42,name="regression analysis",
description="",
parent=method1)
db.session.add(method2)
- method2 = Method(id=33,name="time-series analysis",
+ method2 = Method(id=43,name="time-series analysis",
description="",
parent=method1)
db.session.add(method2)
- method2 = Method(id=34,name="agent-based modeling",
+ method2 = Method(id=44,name="agent-based modeling",
description="",
parent=method1)
db.session.add(method2)
- method2 = Method(id=35,name="digital communication",
+ method2 = Method(id=45,name="digital communication",
description="",
parent=method1)
db.session.add(method2)
- method1 = Method(id=36,name="social complexity modeling/ social simulation",
+ method1 = Method(id=46,name="social complexity modeling/ social simulation",
description="",
parent=method)
db.session.add(method1)
+ reference = Reference(name="Rogers2013",
+ cited="Rogers, R. (2013). Digital methods. Cambridge, Massachusetts, London, England: The MIT Press.")
+ db.session.add(reference)
+ reference = Reference(name="Jünger2018",
+ cited="Jünger, Jakob (2018): Mapping the Field of Automated Data Collection on the Web. Data Types, Collection Approaches and their Research Logic. In: Stützer, Cathleen / Welker, Martin / Egger, Marc (Hg). Computational Social Science in the Age of Big Data. Concepts, Methodologies, Tools, and Applications. Neue Schriften zur Online-Forschung der Deutschen Gesellschaft für Online-Forschung (DGOF). Köln: Halem-Verlag, S. 104-130.")
+ db.session.add(reference)
+ reference = Reference(name="Han_etal2012",
+ cited="Han, J., Kamber, M., & Pei, J. (2012). Data Mining: Concepts and Techniques. Saint Louis, UNITED STATES: Elsevier Science & Technology.")
+ db.session.add(reference)
+ reference = Reference(name="Ignatow_etal2017",
+ cited="Ignatow, G., & Mihalcea, R. F. (2017). Text mining: A guidebook for the social sciences. Los Angeles, London, New Delhi, Singapore, Washington DC, Melbourne: Sage.")
+ db.session.add(reference)
+ reference = Reference(name="Wickham_etal2017",
+ cited="Wickham, H., & Grolemund, G. (2017). R for Data Science: Import, tidy, transform, visualize, and model data. Beijing, Boston, Farnham, Sebastopol, Tokyo: O’Reilly UK Ltd.")
+ db.session.add(reference)
+
+
+
lic_unknown = License(name="Unknown")
lic_bsd = License(name="BSD")
lic_gpl2 = License(name="GPL", version="2.0")
@@ -1345,7 +1420,7 @@ if __name__ == '__main__':
# Build a sample db on the fly, if one does not exist yet.
app_dir = op.realpath(os.path.dirname(__file__))
database_path = op.join(app_dir, app.config['DATABASE_FILE'])
- if not os.path.exists(database_path):
- build_sample_db()
+ #if not os.path.exists(database_path):
+ build_sample_db()
# Start app
app.run(debug=True)
diff --git a/sample_db.sqlite b/sample_db.sqlite
index 4b40f87ea9ff54c043f2eef9a97d635556d650e5..1f327e1a1de8264c04b8213dac76cadc1cc3f59d 100644
Binary files a/sample_db.sqlite and b/sample_db.sqlite differ
diff --git a/templates/admin/index.html b/templates/admin/index.html
index c361a89a482149a4901cb37a98b9ce064a88a8f9..97951f015fccff504af3b94127087cf10e094e6a 100644
--- a/templates/admin/index.html
+++ b/templates/admin/index.html
@@ -4,9 +4,9 @@
<div class="container">
<div class="row">
<div class="col-sm-10 col-sm-offset-1">
- <h1>Digitale Methoden WiSo-Forschungslabor</h1>
+ <h1>Digital methods WiSo-Forschungslabor</h1>
<p class="lead">
- Weitere Infos zum Projekt finden sich <a href="https://www.wiso.uni-hamburg.de/forschung/forschungslabor/aktuelleprojekte/digitale-methoden.html" target="_blank">hier</a>
+ For more information, refer to <a href="https://www.wiso.uni-hamburg.de/forschung/forschungslabor/aktuelleprojekte/digitale-methoden.html" target="_blank">the project website</a>.
</p>
<!--<a class="btn btn-primary" href="/"><i class="glyphicon glyphicon-chevron-left"></i> Back</a>-->
</div>
diff --git a/templates/export/MethodsList.jinja2 b/templates/export/MethodsList.jinja2
index 701d368f9a4490979d53982d2784c15f710b1996..78ac8dad15025308f37aa3ff735fb7e8b6c477c8 100644
--- a/templates/export/MethodsList.jinja2
+++ b/templates/export/MethodsList.jinja2
@@ -5,6 +5,16 @@
:sectnumlevels: 8
{% for method in methods %}
-{% for l in range(method.level+1) %}={% endfor %} {{ method.name }}
+
+{% for l in range(method.level+1) %}={% endfor %} {{ method.name }}{% if method.level==0 %}
+
+---
+{% endif %}
{{method.description}}
+{% endfor %}
+
+[bibliography]
+== References
+{% for reference in references %}
+- [[[{{ reference.name }}]]] {{ reference.cited }}
{% endfor %}
\ No newline at end of file