Skip to main content
Sign in
Snippets Groups Projects
Commit 7fcef7ae authored by Lange, Dr. Herbert's avatar Lange, Dr. Herbert
Browse files

include corpus name and function parameters in the url. change bucket name in the report

parent 2f42ffff
No related branches found
No related tags found
No related merge requests found
...@@ -5,6 +5,8 @@ import java.io.File; ...@@ -5,6 +5,8 @@ import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.lang.Thread; import java.lang.Thread;
import com.fasterxml.jackson.databind.JsonNode;
import jakarta.ws.rs.GET; import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path; import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces; import jakarta.ws.rs.Produces;
...@@ -17,12 +19,16 @@ import jakarta.ws.rs.core.Response; ...@@ -17,12 +19,16 @@ import jakarta.ws.rs.core.Response;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*; import java.util.*;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import de.uni_hamburg.corpora.*; import de.uni_hamburg.corpora.*;
import org.apache.commons.io.FileUtils;
import org.exmaralda.partitureditor.jexmaralda.JexmaraldaException; import org.exmaralda.partitureditor.jexmaralda.JexmaraldaException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -30,7 +36,7 @@ import org.xml.sax.SAXException; ...@@ -30,7 +36,7 @@ import org.xml.sax.SAXException;
/** /**
* @author bba1792 Dr. Herbert Lange * @author bba1792 Dr. Herbert Lange
* @version 20210708 * @version 20220405
* Worker thread for the corpus checker * Worker thread for the corpus checker
*/ */
class CorpusThread extends Thread { class CorpusThread extends Thread {
...@@ -38,6 +44,7 @@ class CorpusThread extends Thread { ...@@ -38,6 +44,7 @@ class CorpusThread extends Thread {
private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); private final Logger logger = LoggerFactory.getLogger(this.getClass().getName());
de.uni_hamburg.corpora.Report report = new de.uni_hamburg.corpora.Report(); de.uni_hamburg.corpora.Report report = new de.uni_hamburg.corpora.Report();
String corpusName;
String inFile ; String inFile ;
String functionNames; String functionNames;
String outFile; String outFile;
...@@ -45,8 +52,9 @@ class CorpusThread extends Thread { ...@@ -45,8 +52,9 @@ class CorpusThread extends Thread {
String token; // Identifier to be sent back to the server to identify and authorize task String token; // Identifier to be sent back to the server to identify and authorize task
String callbackUrl; // URL to be called when the task is done, giving an empty string means skipping the callback String callbackUrl; // URL to be called when the task is done, giving an empty string means skipping the callback
CorpusThread(String infile, String outfile, String functions, Properties properties, String token, CorpusThread(String name, String infile, String outfile, String functions, Properties properties, String token,
String callbackUrl) { String callbackUrl) {
this.corpusName=name;
if (infile.equals("tmp")) if (infile.equals("tmp"))
this.inFile = System.getProperty("java.io.tmpdir") + "/corpus-files"; this.inFile = System.getProperty("java.io.tmpdir") + "/corpus-files";
else else
...@@ -59,17 +67,28 @@ class CorpusThread extends Thread { ...@@ -59,17 +67,28 @@ class CorpusThread extends Thread {
this.outFile = outfile; this.outFile = outfile;
this.token = token ; this.token = token ;
this.callbackUrl = callbackUrl ; this.callbackUrl = callbackUrl ;
logger.info("Input: " + this.inFile + " output: " + this.outFile + " functions: " + functions + " params: " +
this.props + " token: " + this.token + " callback: " + this.callbackUrl);
} }
public void run() { public void run() {
ArrayList<String> functionList = new ArrayList<>(Arrays.asList(functionNames.split(","))); ArrayList<String> functionList = new ArrayList<>(Arrays.asList(functionNames.split(",")));
Set<String> allFunctions = CorpusServices.getCorpusFunctions() ; Set<String> allFunctions = CorpusServices.getCorpusFunctions() ;
CorpusIO cio = new CorpusIO(); CorpusIO cio = new CorpusIO();
report.addNote("CorpusWebServices","Starting run at " +
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").format(LocalDateTime.now()));
try { try {
// Create corpus from given input file/folder // Create corpus from given input file/folder
Corpus corpus = new Corpus(cio.read(new File(inFile).toURI().toURL(),report)) ; Corpus corpus;
if (corpusName != null && !corpusName.isEmpty()) {
URL baseUrl = new File(inFile).toURI().toURL();
corpus = new Corpus(corpusName,baseUrl,cio.read(baseUrl,report)) ;
}
else
corpus = new Corpus(cio.read(new File(inFile).toURI().toURL(),report)) ;
logger.info("Loaded " + corpus.getCorpusData().size() + " corpus files"); logger.info("Loaded " + corpus.getCorpusData().size() + " corpus files");
logger.info("Got report: " + report.getFullReports());
// For all functions to be applied, get their canonical name and create an object for them // For all functions to be applied, get their canonical name and create an object for them
Set<CorpusFunction> functions = new HashSet<>() ; Set<CorpusFunction> functions = new HashSet<>() ;
for (String function : functionList) { for (String function : functionList) {
...@@ -84,24 +103,24 @@ class CorpusThread extends Thread { ...@@ -84,24 +103,24 @@ class CorpusThread extends Thread {
} }
catch (IllegalArgumentException | NoSuchMethodException | InstantiationException | InvocationTargetException | IllegalAccessException e) { catch (IllegalArgumentException | NoSuchMethodException | InstantiationException | InvocationTargetException | IllegalAccessException e) {
logger.warn("Error creating {}", canonical); logger.warn("Error creating {}", canonical);
report.addWarning("CorpusService", "Test " + function + " cannot be created"); report.addWarning("CorpusWebServices", "Test " + function + " cannot be created");
// e.printStackTrace(); // e.printStackTrace();
} }
} }
} }
if (!found) { if (!found) {
// Warn if we could not find the function // Warn if we could not find the function
report.addWarning("CorpusService", "Test " + function + " is not available"); report.addWarning("CorpusWebServices", "Test " + function + " is not available");
logger.warn("Function {} is not available in corpus services", function); logger.warn("Function {} is not available in corpus services", function);
} }
} }
for (CorpusFunction f : functions) { for (CorpusFunction f : functions) {
logger.warn("Running function {}", f.getFunction()); logger.warn("Running function {}", f.getFunction());
report.addNote("CorpusService", "Run test " + f.getFunction()); report.addNote("CorpusWebServices", "Run test " + f.getFunction());
de.uni_hamburg.corpora.Report result = f.execute(corpus); de.uni_hamburg.corpora.Report result = f.execute(corpus);
report.merge(result); report.merge(result);
report.addNote("CorpusService", "Finish test " + f.getFunction()); report.addNote("CorpusWebServices", "Finish test " + f.getFunction());
logger.warn("Done with function {}", f.getFunction()); logger.warn("Done with function {}", f.getFunction());
} }
} catch (URISyntaxException | ClassNotFoundException | IOException | SAXException | JexmaraldaException e) { } catch (URISyntaxException | ClassNotFoundException | IOException | SAXException | JexmaraldaException e) {
...@@ -109,7 +128,8 @@ class CorpusThread extends Thread { ...@@ -109,7 +128,8 @@ class CorpusThread extends Thread {
} }
logger.info("Done with all functions"); logger.info("Done with all functions");
report.addNote("CorpusService","Finished all tests"); report.addNote("CorpusWebServices","Finished all tests at " +
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").format(LocalDateTime.now()));
logger.info("Creating report"); logger.info("Creating report");
// Get summary // Get summary
HashMap<ReportItem.Severity,Integer> summary = CorpusServices.generateSummary(report); HashMap<ReportItem.Severity,Integer> summary = CorpusServices.generateSummary(report);
...@@ -181,7 +201,8 @@ public class CorpusChecker { ...@@ -181,7 +201,8 @@ public class CorpusChecker {
@GET @GET
@Produces(MediaType.TEXT_PLAIN) @Produces(MediaType.TEXT_PLAIN)
public Response checkCorpus(@QueryParam("input") String input, public Response checkCorpus(@QueryParam("name") String name,
@QueryParam("input") String input,
@QueryParam("output") String output, @QueryParam("output") String output,
@QueryParam("functions") String functions, @QueryParam("functions") String functions,
@QueryParam("params") String paramStr, @QueryParam("params") String paramStr,
...@@ -220,11 +241,25 @@ public class CorpusChecker { ...@@ -220,11 +241,25 @@ public class CorpusChecker {
return Response.status(400).entity("400 - " + errorMsg).build(); return Response.status(400).entity("400 - " + errorMsg).build();
} }
Properties params = new Properties(); Properties params = new Properties();
if (paramStr != null) { if (paramStr != null && !paramStr.equals("{}")) {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
params.putAll(mapper.convertValue(paramStr,Map.class)); try {
JsonNode node = mapper.readTree(paramStr);
// Copy the jason data into the properties
for (Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> e = it.next();
params.put(e.getKey(),e.getValue().textValue());
}
logger.info(params.toString());
//params.putAll(mapper.convertValue(paramStr,Map.class));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
else {
params = new Properties();
} }
CorpusThread ct = new CorpusThread(input,output,functions,params,token,callbackUrl); CorpusThread ct = new CorpusThread(name,input,output,functions,params,token,callbackUrl);
ct.start(); ct.start();
Main.addThread(ct); Main.addThread(ct);
return Response.ok().entity("Executing " + functions + " on " + input + return Response.ok().entity("Executing " + functions + " on " + input +
... ...
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment