more bugfixes and tests

verarbeitung/start_script.py → start_script.py

@@ -9,4 +9,4 @@ doi_list.append('https://pubs.acs.org/doi/10.1021/acs.jcim.9b00249')
 #doi_list.append('https://doi.org/10.1021/acs.jcim.9b00249')
 doi_list.append('https://pubs.acs.org/doi/10.1021/acs.jcim.1c00203')
 doi_list.append('https://doi.org/10.1021/acs.jmedchem.0c01332')
-Processing(doi_list, 2, 2, 'test.json')
\ No newline at end of file
+Processing(doi_list, 3, 2, 'test.json')
\ No newline at end of file

verarbeitung/construct_new_graph/add_citations_rec.py

@@ -32,10 +32,12 @@ def get_cit_type_list(pub, cit_type):
 
         function to create nodes and edges and call create_graph_structure_citations
     '''
-    if (cit_type == "Citation"):
+    if cit_type == "Citation":
         return(pub.citations)
-    else:
+    elif cit_type == "Reference":
         return(pub.references)
+    else:
+        return(ValueError)
 
 def create_global_lists_cit(input_nodes, input_edges, pub, search_depth, search_depth_max, cit_type, test_var):
     '''

verarbeitung/construct_new_graph/initialize_graph.py

@@ -15,6 +15,7 @@ __status__ = "Production"
 
 
 import sys  
+import gc
 from pathlib import Path
 from os import error
 sys.path.append("../")
@@ -24,6 +25,43 @@ from verarbeitung.get_pub_from_input import get_pub
 from .export_to_json import output_to_json
 from .add_citations_rec import add_citations, create_global_lists_cit
 
+def initialize_nodes_list_test(doi_input_list, search_depth_max, search_height_max, test_var):
+    '''
+        :param doi_input_list:      input list of doi from UI
+        :type doi_input_list:       List[String]
+
+        :param search_depth_max:    maximum depth to search for references
+        :type search_depth_max:     int
+
+        :param search_height_max:   maximum height to search for citations
+        :type search_height_max:    int
+
+        :param test_var:            variable to differenciate between test and url call
+        :type test_var:             boolean
+
+        for unit test purposes only
+    '''
+    global nodes, edges
+    nodes = []
+    edges = []
+    return(initialize_nodes_list(doi_input_list, search_depth_max, search_height_max, test_var))
+
+def complete_inner_edges_test(test_nodes, test_edges):
+    '''
+        :param test_nodes:      list of publications from unit test
+        :type test_nodes:       List[Publication]
+
+        :param test_nodes:      list of links from unit test
+        :type test_nodes:       List[List[String,String]]
+
+        for unit test purposes only
+        '''
+        
+    global nodes, edges
+    nodes = test_nodes
+    edges = test_edges
+    complete_inner_edges()
+    return(nodes, edges)
 
 def initialize_nodes_list(doi_input_list, search_depth_max, search_height_max, test_var):
     '''
@@ -54,6 +92,7 @@ def initialize_nodes_list(doi_input_list, search_depth_max, search_height_max, t
 
         # checks if publication already exists in nodes
         not_in_nodes = True #boolean value to check if a node already exists in the set of nodes
+
         for node in nodes: #iterates over every node in the set of nodes
             if (pub.doi_url == node.doi_url): #determines that a node with this doi already is in the set
                 not_in_nodes = False #false --> node will not be created
@@ -110,6 +149,15 @@ def init_graph_construction(doi_input_list, search_depth, search_height, test_va
         :param test_var:        variable to differenciate between test and url call
         :type test_var:         boolean
 
+        :param test_var:        variable to check if call is from update_graph with known nodes and edges or fresh construction
+        :type test_var:         boolean
+
+        :param input_nodes:     list of publications from update_graph
+        :type input_nodes:      List[Publication]
+
+        :param input_nodes:     list of links from update_graph
+        :type input_nodes:      List[List[String,String]]
+
         main function to start graph generation
     '''
 
@@ -145,7 +193,11 @@ def init_graph_construction(doi_input_list, search_depth, search_height, test_va
     # adds edges between reference group and citation group of known publications
     complete_inner_edges()
 
-    # calls a skript to save nodes and edges of graph in .json file
-    output_to_json(nodes, edges, test_var = test_var)
+    # garbage collection to delete nodes and edges lists. Needed because python keeps lists after function end till next function call
+    new_nodes = nodes.copy()
+    new_edges = edges.copy()
+    del nodes
+    del edges
+    gc.collect()
 
-    return(nodes,edges)
+    return(new_nodes,new_edges)

verarbeitung/test/construct_graph_unittest.py

 import unittest
 
 import sys
-from pathlib import Path
+  
 
 sys.path.append("../")
 
-from verarbeitung.construct_new_graph.initialize_graph import init_graph_construction, initialize_nodes_list
+from verarbeitung.construct_new_graph.initialize_graph import init_graph_construction, initialize_nodes_list_test, complete_inner_edges_test
+from verarbeitung.construct_new_graph.add_citations_rec import get_cit_type_list
+from verarbeitung.test.input_test import input_test_func
 
 class ConstructionTest(unittest.TestCase):
      maxDiff = None
 
 
      def testCycle(self):
-         nodes, edges = init_graph_construction(['doiz1'],1,1,True)
+         nodes, edges = init_graph_construction(['doiz1'],1,1,True,False)
          doi_nodes = keep_only_dois(nodes)
          self.assertCountEqual(doi_nodes, ['doiz1', 'doiz2'])
          self.assertCountEqual(edges, [['doiz1', 'doiz2'], ['doiz2', 'doiz1']])
 
-         nodes, edges = init_graph_construction(['doiz1'],2,2,True)
+         nodes, edges = init_graph_construction(['doiz1'],2,2,True,False)
          doi_nodes = keep_only_dois(nodes)
          self.assertCountEqual(doi_nodes, ['doiz1', 'doiz2'])
          self.assertCountEqual(edges, [['doiz2', 'doiz1'], ['doiz1', 'doiz2']])
@@ -29,56 +31,56 @@ class ConstructionTest(unittest.TestCase):
     #def testEmptyDepth(self):
 
      def testEmptyDepthHeight(self):
-         nodes, edges = init_graph_construction(['doi1'],0,0,True)
+         nodes, edges = init_graph_construction(['doi1'],0,0,True,False)
          doi_nodes = keep_only_dois(nodes)
          self.assertCountEqual(doi_nodes,['doi1'])
          self.assertCountEqual(edges, [])
 
-         nodes, edges = init_graph_construction(['doi1', 'doi2'],0,0,True)
+         nodes, edges = init_graph_construction(['doi1', 'doi2'],0,0,True,False)
          doi_nodes = keep_only_dois(nodes)
          self.assertCountEqual(doi_nodes, ['doi1','doi2'])
          self.assertCountEqual(edges, [['doi1', 'doi2']])
 
-         nodes, edges = init_graph_construction(['doi1', 'doi2', 'doi3'],0,0,True)
+         nodes, edges = init_graph_construction(['doi1', 'doi2', 'doi3'],0,0,True,False)
          doi_nodes = keep_only_dois(nodes)
          self.assertCountEqual(doi_nodes, ['doi1','doi2', 'doi3'])
          self.assertCountEqual(edges, [['doi3', 'doi1'], ['doi1', 'doi2']])
 
 
      def testInnerEdges(self):
-        nodes, edges = init_graph_construction(['doi_ie1'],1,1,True)
+        nodes, edges = init_graph_construction(['doi_ie1'],1,1,True,False)
         doi_nodes = keep_only_dois(nodes)
         self.assertCountEqual(doi_nodes,['doi_ie1','doi_ie2','doi_ie3'])
         self.assertCountEqual(edges,[['doi_ie1','doi_ie2'],['doi_ie3','doi_ie1'],['doi_ie3','doi_ie2']])
      
      def testRightHeight(self):
-          nodes, edges = init_graph_construction(['doi_h01'],0,1,True)
+          nodes, edges = init_graph_construction(['doi_h01'],0,1,True,False)
           doi_nodes = keep_only_dois(nodes)
           self.assertCountEqual(doi_nodes,['doi_h01'])
           self.assertCountEqual(edges, [])
 
-          nodes, edges = init_graph_construction(['doi_h02'],0,1,True)
+          nodes, edges = init_graph_construction(['doi_h02'],0,1,True,False)
           doi_nodes = keep_only_dois(nodes)
           self.assertCountEqual(doi_nodes,['doi_h02','doi_h1'])
           self.assertCountEqual(edges, [['doi_h1','doi_h02']])
 
-          nodes, edges = init_graph_construction(['doi_h02'],0,2,True)
+          nodes, edges = init_graph_construction(['doi_h02'],0,2,True,False)
           doi_nodes = keep_only_dois(nodes)
           self.assertCountEqual(doi_nodes,['doi_h02','doi_h1','doi_h2'])
           self.assertCountEqual(edges, [['doi_h1','doi_h02'], ['doi_h2','doi_h1']])
 
      def testRightDepth(self):
-          nodes, edges = init_graph_construction(['doi_d01'],1,0,True)
+          nodes, edges = init_graph_construction(['doi_d01'],1,0,True,False)
           doi_nodes = keep_only_dois(nodes)
           self.assertCountEqual(doi_nodes,['doi_d01'])
           self.assertCountEqual(edges, [])
 
-          nodes, edges = init_graph_construction(['doi_d02'],1,0,True)
+          nodes, edges = init_graph_construction(['doi_d02'],1,0,True,False)
           doi_nodes = keep_only_dois(nodes)
           self.assertCountEqual(doi_nodes,['doi_d02','doi_d1'])
           self.assertCountEqual(edges, [['doi_d02','doi_d1']])
 
-          nodes, edges = init_graph_construction(['doi_d02'],2,0,True)
+          nodes, edges = init_graph_construction(['doi_d02'],2,0,True,False)
           doi_nodes = keep_only_dois(nodes)
           self.assertCountEqual(doi_nodes,['doi_d02','doi_d1','doi_d2'])
           self.assertCountEqual(edges, [['doi_d02','doi_d1'], ['doi_d1','doi_d2']])
@@ -89,19 +91,54 @@ class ConstructionTest(unittest.TestCase):
      # initialize_graph.py:
 
      def test_initialize_nodes_list(self):
-          references_pub_obj_list, citations_pub_obj_list = initialize_nodes_list(['doi_lg_1_i','doi_lg_2_i'], 0, 0, True)
+          references_pub_obj_list, citations_pub_obj_list = initialize_nodes_list_test(['doi_lg_1_i','doi_lg_2_i'], 0, 0, True)
           doi_references = keep_only_dois(references_pub_obj_list)
           doi_citations = keep_only_dois(citations_pub_obj_list)
           self.assertCountEqual(doi_references, [])
           self.assertCountEqual(doi_citations, [])
 
-          references_pub_obj_list, citations_pub_obj_list = initialize_nodes_list(['doi_lg_1_i','doi_lg_2_i'], 1, 1, True)
+          references_pub_obj_list, citations_pub_obj_list = initialize_nodes_list_test(['doi_lg_1_i','doi_lg_2_i'], 1, 1, True)
           doi_references = keep_only_dois(references_pub_obj_list)
           doi_citations = keep_only_dois(citations_pub_obj_list)
           self.assertCountEqual(doi_references, ['doi_lg_1_d11','doi_lg_1_d12','doi_lg_2_d11','doi_lg_2_d12'])
           self.assertCountEqual(doi_citations, ['doi_lg_1_h11','doi_lg_1_h12','doi_cg_i','doi_lg_2_h11','doi_lg_2_h12'])
 
-
+     def test_complete_inner_edges(self):
+          pub_lg_1_i = input_test_func('doi_lg_1_i')
+          pub_lg_1_i.group = 0
+          pub_lg_1_h_12 = input_test_func('doi_lg_1_h12')
+          pub_lg_1_h_12.group = 1
+          pub_lg_1_d_12 = input_test_func('doi_lg_1_d12')
+          pub_lg_1_d_12.group = -1
+          nodes = [pub_lg_1_i, pub_lg_1_h_12, pub_lg_1_d_12]
+          edges = [['doi_lg_1_i','doi_lg_1_d12'],['doi_lg_1_h12','doi_lg_1_i']]
+          processed_nodes, processed_edges = complete_inner_edges_test(nodes, edges)
+          self.assertCountEqual(processed_nodes, [pub_lg_1_i, pub_lg_1_h_12, pub_lg_1_d_12])
+          self.assertCountEqual(processed_edges, [['doi_lg_1_i','doi_lg_1_d12'],['doi_lg_1_h12','doi_lg_1_i'],['doi_lg_1_h12','doi_lg_1_d12']])
+
+     #    add_citations_rec.py:
+
+     def test_get_type_list(self):
+          pub_lg_1_i = input_test_func('doi_lg_1_i')
+          pub_lg_1_i.group = 0
+          self.assertEqual(get_cit_type_list(pub_lg_1_i, "Hallo"), ValueError)
+          
+          pub_lg_1_h_12 = input_test_func('doi_lg_1_h12')
+          pub_lg_1_h_12.group = 1
+          pub_lg_1_h_12_refs = get_cit_type_list(pub_lg_1_h_12, "Reference")
+          pub_lg_1_h_12_cits = get_cit_type_list(pub_lg_1_h_12, "Citation")
+          self.assertCountEqual(keep_only_dois(pub_lg_1_h_12_refs), keep_only_dois(pub_lg_1_h_12.references))
+          self.assertCountEqual(keep_only_dois(pub_lg_1_h_12_cits), keep_only_dois(pub_lg_1_h_12.citations))
+          
+          pub_lg_1_d_12 = input_test_func('doi_lg_1_d12')
+          pub_lg_1_d_12.group = -1
+          pub_lg_1_d_12_refs = get_cit_type_list(pub_lg_1_d_12, "Reference")
+          pub_lg_1_d_12_cits = get_cit_type_list(pub_lg_1_d_12, "Citation")
+          self.assertCountEqual(keep_only_dois(pub_lg_1_d_12_refs), keep_only_dois(pub_lg_1_d_12.references))
+          self.assertCountEqual(keep_only_dois(pub_lg_1_d_12_cits), keep_only_dois(pub_lg_1_d_12.citations))
+
+     def test_create_graph_structure_citations(self):
+          print("Hallo")
 
 
 def keep_only_dois(nodes):

verarbeitung/test_output.json

-{"nodes": [{"doi": "doi_lg_1_i", "name": "title_lg_1_i", "author": ["contributor_lg_1_i"], "year": "date_lg_1_i", "journal": "journal_lg_1_i", "group": "Input", "depth": 0, "citations": 2}, {"doi": "doi_lg_1_d11", "name": "title_lg_1_d11", "author": ["contributor_lg_1_d11"], "year": "date_lg_1_d11", "journal": "journal_lg_1_d11", "group": "Reference", "depth": -1, "citations": 1}, {"doi": "doi_lg_1_d12", "name": "title_lg_1_d12", "author": ["contributor_lg_1_d12"], "year": "date_lg_1_d12", "journal": "journal_lg_1_d12", "group": "Reference", "depth": -1, "citations": 2}, {"doi": "doi_lg_1_h11", "name": "title_lg_1_h11", "author": ["contributor_lg_1_h11"], "year": "date_lg_1_h11", "journal": "journal_lg_1_h11", "group": "Citedby", "depth": 1, "citations": 2}, {"doi": "doi_lg_1_h12", "name": "title_lg_1_h12", "author": ["contributor_lg_1_h12"], "year": "date_lg_1_h12", "journal": "journal_lg_1_h12", "group": "Citedby", "depth": 1, "citations": 2}, {"doi": "doi_lg_1_h21", "name": "title_lg_1_h21", "author": ["contributor_lg_1_h21"], "year": "date_lg_1_h21", "journal": "journal_lg_1_h21", "group": "Citedby", "depth": 2, "citations": 0}, {"doi": "doi_lg_1_h22", "name": "title_lg_1_h22", "author": ["contributor_lg_1_h22"], "year": "date_lg_1_h22", "journal": "journal_lg_1_h22", "group": "Citedby", "depth": 2, "citations": 0}, {"doi": "doi_lg_1_h23", "name": "title_lg_1_h23", "author": ["contributor_lg_1_h23"], "year": "date_lg_1_h23", "journal": "journal_lg_1_h23", "group": "Citedby", "depth": 2, "citations": 0}, {"doi": "doi_lg_1_d21", "name": "title_lg_1_d21", "author": ["contributor_lg_1_d21"], "year": "date_lg_1_d21", "journal": "journal_lg_1_d21", "group": "Reference", "depth": -2, "citations": 2}, {"doi": "doi_lg_1_d22", "name": "title_lg_1_d22", "author": ["contributor_lg_1_d22"], "year": "date_lg_1_d22", "journal": "journal_lg_1_d22", "group": "Reference", "depth": -2, "citations": 2}, {"doi": "doi_lg_1_d23", "name": "title_lg_1_d23", "author": ["contributor_lg_1_d23"], "year": "date_lg_1_d23", "journal": "journal_lg_1_d23", "group": "Reference", "depth": -2, "citations": 2}], "links": [{"source": "doi_lg_1_i", "target": "doi_lg_1_d11"}, {"source": "doi_lg_1_i", "target": "doi_lg_1_d12"}, {"source": "doi_lg_1_h11", "target": "doi_lg_1_i"}, {"source": "doi_lg_1_h12", "target": "doi_lg_1_i"}, {"source": "doi_lg_1_h21", "target": "doi_lg_1_h11"}, {"source": "doi_lg_1_h22", "target": "doi_lg_1_h11"}, {"source": "doi_lg_1_h22", "target": "doi_lg_1_h12"}, {"source": "doi_lg_1_h23", "target": "doi_lg_1_h12"}, {"source": "doi_lg_1_d11", "target": "doi_lg_1_d21"}, {"source": "doi_lg_1_d11", "target": "doi_lg_1_d22"}, {"source": "doi_lg_1_d21", "target": "doi_lg_1_d22"}, {"source": "doi_lg_1_d22", "target": "doi_lg_1_d21"}, {"source": "doi_lg_1_d12", "target": "doi_lg_1_d23"}, {"source": "doi_lg_1_h12", "target": "doi_lg_1_d12"}]}
\ No newline at end of file
+{"nodes": [{"doi": "doi_h02", "name": "title_h02", "author": ["contributor_h02"], "year": "date_h02", "journal": "journal_h02", "group": "Input", "depth": 0, "citations": 1}, {"doi": "doi_h1", "name": "title_h1", "author": ["contributor_h1"], "year": "date_h1", "journal": "journal_h1", "group": "Citedby", "depth": 1, "citations": 1}, {"doi": "doi_h2", "name": "title_h2", "author": ["contributor_h2"], "year": "date_h2", "journal": "journal_h2", "group": "Citedby", "depth": 2, "citations": 1}], "links": [{"source": "doi_h1", "target": "doi_h02"}, {"source": "doi_h2", "target": "doi_h1"}]}
\ No newline at end of file

verarbeitung/update_graph/connect_new_input.py

@@ -19,7 +19,6 @@ from os import error
 
 sys.path.append("../")
 
-from .import_from_json import input_from_json
 from verarbeitung.construct_new_graph.initialize_graph import init_graph_construction
 from verarbeitung.construct_new_graph.add_citations_rec import add_citations, get_cit_type_list, create_global_lists_cit
 

verarbeitung/update_graph/update_depth.py

@@ -72,7 +72,10 @@ def get_old_max_references(old_depth):
     old_max_references = []
     for pub in processed_input_list:
         if (abs(pub.group) == old_depth):
-            old_max_references.append(pub.references)
+            for reference in pub.references:
+                for ref_pub in processed_input_list:
+                    if reference.doi_url == ref_pub.doi_url:
+                        old_max_references.append(ref_pub)
     return(old_max_references)
 
 def get_old_max_citations(old_height):
@@ -85,7 +88,10 @@ def get_old_max_citations(old_height):
     old_max_citations = []
     for pub in processed_input_list:
         if (abs(pub.group) == old_height):
-            old_max_citations.append(pub.citations)
+            for citation in pub.citations:
+                for cit_pub in processed_input_list:
+                    if citation.doi_url == cit_pub.doi_url:
+                        old_max_citations.append(cit_pub)
     return(old_max_citations)
 
 def update_depth(obj_input_list, input_edges, new_depth, new_height, test_var):
@@ -123,10 +129,10 @@ def update_depth(obj_input_list, input_edges, new_depth, new_height, test_var):
     
     # adds publications and links for new recursion levels
     elif (old_depth < new_depth):
-        old_max_references = get_old_max_references()
+        old_max_references = get_old_max_references(old_depth)
         add_citations(processed_input_list, valid_edges, old_max_references, old_depth+1, new_depth, "Reference", test_var)
     elif (old_height < new_height):
-        old_max_citations = get_old_max_citations()
+        old_max_citations = get_old_max_citations(old_height)
         add_citations(processed_input_list, valid_edges, old_max_citations, old_height+1, new_height, "Citation", test_var)
     back_to_valid_edges(valid_edges, processed_input_list)
 

verarbeitung/update_graph/update_graph.py

@@ -20,7 +20,6 @@ sys.path.append("../../")
 
 from input.publication import Publication
 from verarbeitung.get_pub_from_input import get_pub
-from verarbeitung.construct_new_graph.initialize_graph import init_graph_construction
 from .Knoten_Vergleich import doi_listen_vergleichen
 from .delete_nodes_edges import delete_nodes_and_edges
 from .connect_new_input import connect_old_and_new_input