Introduce config JSON files and class

common/config.py

+"""
+Contains a class that handles endpoint configuration. Its main
+functions are reading the configuration files and restoring the
+defaults if some keys are absent there.
+"""
+from .enums import *
+import json
+import copy
+import re
+import os
+
+
+class ResourceConfig:
+    """
+    Properties of this class correspond to the keys in a configuration file
+    for one of the resources this endpoint communicates with.
+    """
+
+    def __init__(self, fnameConfig=None):
+        self.platform = CorpPlatform.annis
+        self.corpus_id = ''
+        self.transport_protocol = 'https'
+        self.host = '127.0.0.1'
+        self.port = '5000'
+        self.url_path = '127.0.0.1'
+        self.titles = []
+        self.descriptions = []
+        self.authors = []
+        self.contacts = []
+        self.extents = []
+        self.history = []
+        self.restrictions = []
+        self.max_hits = 10
+        self.basic_search_capability = True
+        self.advanced_search_capability = False
+        self.hits_supported = True
+        self.adv_supported = False
+        self.supported_layers = []
+        self.resources = []
+
+        self.query_timeout = 60
+
+        self.boolParams = set(k for k in self.__dict__
+                              if type(self.__dict__[k]) == bool)
+        self.intParams = set(k for k in self.__dict__
+                             if type(self.__dict__[k]) == int)
+        self.lsParams = {}
+
+        # dictionaries where values are strings
+        self.dict_sParams = {}
+
+        # dictionaries where values are lists of strings
+        self.dict_lsParams = {}
+
+        # dictionaries where values are dictionaries {k: string}
+        self.dict_dParams = {}
+
+        if fnameConfig is not None and os.path.exists(fnameConfig):
+            self.load_settings(fnameConfig)
+
+
+    def load_settings(self, fnameConfig):
+        """
+        Load configuration for one of the resources from a JSON file.
+        """
+        with open(fnameConfig, 'r', encoding='utf-8') as fConfig:
+            config = json.load(fConfig)
+        for k, v in config.items():
+            setattr(self, k, v)
+
+    def as_dict(self):
+        """
+        Return current settings as a dictionary.
+        """
+        dictSettings = copy.deepcopy(vars(self))
+        for k in [_ for _ in dictSettings.keys()]:
+            if dictSettings[k] is None:
+                dictSettings[k] = ''
+        return dictSettings
+
+    def gui_str_to_dict(self, s, value_type='list'):
+        """
+        Process one input string that describes a dictionary.
+        """
+        d = {}
+        s = s.replace('\r', '').strip()
+        s = re.sub('\n\n+', '\n', s, flags=re.DOTALL)
+        if value_type == 'dict':
+            prevKey = ''
+            curData = {}
+            for line in s.split('\n'):
+                if not line.startswith(' '):
+                    curKey = line.strip(': ')
+                    if len(prevKey) > 0 and curKey != prevKey:
+                        d[prevKey] = curData
+                        curData = {}
+                    prevKey = curKey
+                else:
+                    line = line.strip()
+                    if ':' not in line:
+                        continue
+                    k, v = line.split(':')
+                    k = k.rstrip()
+                    v = v.lstrip()
+                    curData[k] = v
+            if len(curData) > 0:
+                d[prevKey] = curData
+        else:
+            for line in s.split('\n'):
+                line = line.strip()
+                if ':' not in line:
+                    continue
+                k, v = line.split(':')
+                k = k.rstrip()
+                v = v.lstrip()
+                if value_type == 'list':
+                    if len(v) <= 0:
+                        v = []
+                    else:
+                        v = [vp.strip() for vp in v.split(',')]
+                d[k] = v
+        return d
+
+    def processed_gui_config(self, data):
+        """
+        Turn form data filled by the user in the configuration GUI to
+        a dictionary in the correct format.
+        """
+        dictConfig = {}
+        for f in self.boolParams:
+            if f in data and len(data[f]) > 0:
+                dictConfig[f] = True
+            else:
+                dictConfig[f] = False
+        for f in self.intParams:
+            if f in data and len(data[f]) > 0:
+                dictConfig[f] = int(data[f])
+        for f in self.lsParams:
+            if f in data and len(data[f]) > 0:
+                dictConfig[f] = [v.strip() for v in data[f].replace('\r', '').strip().split('\n')]
+            else:
+                dictConfig[f] = []
+        for f in self.dict_sParams:
+            if f in data and len(data[f]) > 0:
+                dictConfig[f] = self.gui_str_to_dict(data[f], value_type='string')
+            else:
+                dictConfig[f] = {}
+        for f in self.dict_lsParams:
+            if f in data and len(data[f]) > 0:
+                dictConfig[f] = self.gui_str_to_dict(data[f], value_type='list')
+            else:
+                dictConfig[f] = {}
+        for k, v in data.items():
+            if '%' in k:
+                continue
+            if k not in dictConfig:
+                dictConfig[k] = v
+        return dictConfig
+
+    def save_settings(self, fnameOut, data=None):
+        """
+        Save current or new configuration as a JSON file (can be used to edit
+        configuration files through a web interface).
+        """
+        if data is None or len(data) <= 0:
+            dictConfig = self.as_dict()
+        else:
+            dictConfig = self.processed_gui_config(data)
+        with open(fnameOut, 'w', encoding='utf-8') as fOut:
+            json.dump(dictConfig, fOut, sort_keys=True, ensure_ascii=False, indent=2)

config/test.json

+{
+	"host": "0.0.0.0",
+	"port": "80",
+	"max_hits": 20
+}
\ No newline at end of file

main.py

@@ -6,15 +6,28 @@ from fastapi.responses import JSONResponse
 from common.query_parser import QueryParser
 from common.enums import *
 from common.diagnostics import Diagnostic
+from common.config import ResourceConfig
 import json
+import os
+import re
 import uvicorn
 
+rxExt = re.compile('\\.[^.]*$')
 
 app = FastAPI()
 app.mount('/static', StaticFiles(directory='static'), name='static')
 templates = Jinja2Templates(directory='static')
 
 app.qp = QueryParser()
+app.configs = {}
+i = 0
+for fname in os.listdir('config'):
+    if not fname.lower().endswith('.json'):
+        continue
+    i += 1
+    fnameFull = os.path.join('config', fname)
+    fnameNoExt = rxExt.sub('', fname)
+    app.configs[fnameNoExt] = ResourceConfig(fnameFull)
 
 
 @app.get('/')

static/endpoint_description.xml

     <sru:extraResponseData>
         <ed:EndpointDescription xmlns:ed="http://clarin.eu/fcs/endpoint-description" version="{{ ep_version }}">
-            <ed:Capabilities>{% if capability_basic_search %}
-                <ed:Capability>http://clarin.eu/fcs/capability/basic-search</ed:Capability>{% endif %}{% if capability_advanced_search and ep_version >= 2 %}
+            <ed:Capabilities>{% if config.basic_search_capability %}
+                <ed:Capability>http://clarin.eu/fcs/capability/basic-search</ed:Capability>{% endif %}{% if config.advanced_search_capability and ep_version >= 2 %}
                 <ed:Capability>http://clarin.eu/fcs/capability/advanced-search</ed:Capability>{% endif %}
             </ed:Capabilities>
-            <ed:SupportedDataViews>{% if hits_supported %}
-                <ed:SupportedDataView id="hits" delivery-policy="send-by-default">application/x-clarin-fcs-hits+xml</ed:SupportedDataView>{% endif %}{% if adv_supported and ep_version >= 2%}
+            <ed:SupportedDataViews>{% if config.hits_supported %}
+                <ed:SupportedDataView id="hits" delivery-policy="send-by-default">application/x-clarin-fcs-hits+xml</ed:SupportedDataView>{% endif %}{% if config.adv_supported and ep_version >= 2%}
                 <ed:SupportedDataView id="adv" delivery-policy="send-by-default">application/x-clarin-fcs-adv+xml</ed:SupportedDataView>{% endif %}
-            </ed:SupportedDataViews>{% if capability_advanced_search and ep_version >= 2 %}
-            <ed:SupportedLayers>{% for layer in supported_layers %}
+            </ed:SupportedDataViews>{% if config.advanced_search_capability and ep_version >= 2 %}
+            <ed:SupportedLayers>{% for layer in config.supported_layers %}
                 <ed:SupportedLayer id="{{ layer.id }}" result-id="{{ layer.result_id }}"{% if layer.alt_value_info %} alt-value-info="{{ layer.alt_value_info }}"{% endif %}{% if layer.alt_value_info_uri %} alt-value-info-uri="{{ layer.alt_value_info_uri }}"{% endif %}{% if layer.qualifier %} qualifier="{{ layer.qualifier }}"{% endif %}>{{ layer.layer_type }}</ed:SupportedLayer>{% endfor %}
             </ed:SupportedLayers>
             {% endif %}
-            <ed:Resources>{% for r in resources %}
+            <ed:Resources>{% for r in config.resources %}
                 <ed:Resource pid="{{ r.pid }}">{% for title in r.titles %}
                     <ed:Title xml:lang="{{ title.lang }}">{{ title.content }}</ed:Title>{% endfor %}{% for desc in r.descriptions %}
                     <ed:Description xml:lang="{{ desc.lang }}">{{ desc.content }}</ed:Description>{% if landing_page|length > 0%}

static/explain_response_1.2.xml

@@ -9,18 +9,18 @@
             <zr:explain xmlns:zr="http://explain.z3950.org/dtd/2.0/">
             <!-- <zr:serverInfo > is REQUIRED -->
                 <zr:serverInfo protocol="SRU" version="1.2" transport="{{ transport_protocol }}" method="GET">
-                    <zr:host>{{ host }}</zr:host>
-                    <zr:port>{{ port }}</zr:port>
-                    <zr:database>{{ url_path }}/fcs-endpoint/{{ platform }}/{{ corpus_id }}</zr:database>
+                    <zr:host>{{ config.host }}</zr:host>
+                    <zr:port>{{ config.port }}</zr:port>
+                    <zr:database>{{ config.url_path }}/fcs-endpoint/{{ config.platform }}/{{ config.corpus_id }}</zr:database>
                 </zr:serverInfo>
                 <!-- <zr:databaseInfo> is REQUIRED -->
-                <zr:databaseInfo>{% for title in titles %}
-                    <zr:title lang="{{ title.lang }}"{% if title.primary %} primary="true"{% endif %}>{{ title.content }}</zr:title>{% endfor %}{% for desc in descriptions %}
-                    <zr:description lang="{{ desc.lang }}"{% if desc.primary %} primary="true"{% endif %}>{{ desc.content }}</zr:description>{% endfor %}{% for author in authors %}
-                    <zr:author lang="{{ author.lang }}"{% if author.primary %} primary="true"{% endif %}>{{ author.content }}</zr:author>{% endfor %}{% for contact in contacts %}
-                    <zr:contact lang="{{ contact.lang }}"{% if contact.primary %} primary="true"{% endif %}>{{ contact.content }}</zr:contact>{% endfor %}{% for extent in extents %}
-                    <zr:extent lang="{{ extent.lang }}"{% if extent.primary %} primary="true"{% endif %}>{{ extent.content }}</zr:extent>{% endfor %}{% for hist in history %}
-                    <zr:history lang="{{ hist.lang }}"{% if hist.primary %} primary="true"{% endif %}>{{ hist.content }}</zr:history>{% endfor %}{% for restr in restrictions %}
+                <zr:databaseInfo>{% for title in config.titles %}
+                    <zr:title lang="{{ title.lang }}"{% if title.primary %} primary="true"{% endif %}>{{ title.content }}</zr:title>{% endfor %}{% for desc in config.descriptions %}
+                    <zr:description lang="{{ desc.lang }}"{% if desc.primary %} primary="true"{% endif %}>{{ desc.content }}</zr:description>{% endfor %}{% for author in config.authors %}
+                    <zr:author lang="{{ author.lang }}"{% if author.primary %} primary="true"{% endif %}>{{ author.content }}</zr:author>{% endfor %}{% for contact in config.contacts %}
+                    <zr:contact lang="{{ contact.lang }}"{% if contact.primary %} primary="true"{% endif %}>{{ contact.content }}</zr:contact>{% endfor %}{% for extent in config.extents %}
+                    <zr:extent lang="{{ extent.lang }}"{% if extent.primary %} primary="true"{% endif %}>{{ extent.content }}</zr:extent>{% endfor %}{% for hist in config.history %}
+                    <zr:history lang="{{ hist.lang }}"{% if hist.primary %} primary="true"{% endif %}>{{ hist.content }}</zr:history>{% endfor %}{% for restr in config.restrictions %}
                     <zr:restrictions lang="{{ restr.lang }}"{% if restr.primary %} primary="true"{% endif %}>{{ restr.content }}</zr:restrictions>{% endfor %}
                 </zr:databaseInfo>
                 <!-- <zr:schemaInfo> is REQUIRED -->
@@ -28,10 +28,10 @@
                     <zr:schema identifier="http://clarin.eu/fcs/resource" name="fcs">
                         <zr:title lang="en" primary="true">CLARIN Federated Content Search</zr:title>
                     </zr:schema>
-                </zr:schemaInfo>{% if max_hits > 0 %}
+                </zr:schemaInfo>{% if config.max_hits > 0 %}
                 <!-- <zr:configInfo> is OPTIONAL -->
                 <zr:configInfo>
-                    <zr:setting type="maximumRecords">{{ max_hits }}</zr:setting>
+                    <zr:setting type="maximumRecords">{{ config.max_hits }}</zr:setting>
                 </zr:configInfo>{% endif %}
             </zr:explain>
         </sru:recordData>