Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
corpus-web-services
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Requirements
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Lange, Dr. Herbert
corpus-web-services
Commits
7fcef7ae
Commit
7fcef7ae
authored
2 years ago
by
Lange, Dr. Herbert
Browse files
Options
Downloads
Patches
Plain Diff
include corpus name and function parameters in the url. change bucket name in the report
parent
2f42ffff
No related branches found
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
src/main/java/de/uni_hamburg/corpora/server/CorpusChecker.java
+52
-17
52 additions, 17 deletions
...ain/java/de/uni_hamburg/corpora/server/CorpusChecker.java
with
52 additions
and
17 deletions
src/main/java/de/uni_hamburg/corpora/server/CorpusChecker.java
+
52
−
17
View file @
7fcef7ae
...
...
@@ -5,6 +5,8 @@ import java.io.File;
import
java.io.FileWriter
;
import
java.io.IOException
;
import
java.lang.Thread
;
import
com.fasterxml.jackson.databind.JsonNode
;
import
jakarta.ws.rs.GET
;
import
jakarta.ws.rs.Path
;
import
jakarta.ws.rs.Produces
;
...
...
@@ -17,12 +19,16 @@ import jakarta.ws.rs.core.Response;
import
java.lang.reflect.InvocationTargetException
;
import
java.net.URISyntaxException
;
import
java.net.URL
;
import
java.time.LocalDateTime
;
import
java.time.format.DateTimeFormatter
;
import
java.util.*
;
import
com.fasterxml.jackson.core.JsonProcessingException
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
de.uni_hamburg.corpora.*
;
import
org.apache.commons.io.FileUtils
;
import
org.exmaralda.partitureditor.jexmaralda.JexmaraldaException
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -30,7 +36,7 @@ import org.xml.sax.SAXException;
/**
* @author bba1792 Dr. Herbert Lange
* @version 202
10708
* @version 202
20405
* Worker thread for the corpus checker
*/
class
CorpusThread
extends
Thread
{
...
...
@@ -38,6 +44,7 @@ class CorpusThread extends Thread {
private
final
Logger
logger
=
LoggerFactory
.
getLogger
(
this
.
getClass
().
getName
());
de
.
uni_hamburg
.
corpora
.
Report
report
=
new
de
.
uni_hamburg
.
corpora
.
Report
();
String
corpusName
;
String
inFile
;
String
functionNames
;
String
outFile
;
...
...
@@ -45,8 +52,9 @@ class CorpusThread extends Thread {
String
token
;
// Identifier to be sent back to the server to identify and authorize task
String
callbackUrl
;
// URL to be called when the task is done, giving an empty string means skipping the callback
CorpusThread
(
String
infile
,
String
outfile
,
String
functions
,
Properties
properties
,
String
token
,
CorpusThread
(
String
name
,
String
infile
,
String
outfile
,
String
functions
,
Properties
properties
,
String
token
,
String
callbackUrl
)
{
this
.
corpusName
=
name
;
if
(
infile
.
equals
(
"tmp"
))
this
.
inFile
=
System
.
getProperty
(
"java.io.tmpdir"
)
+
"/corpus-files"
;
else
...
...
@@ -59,17 +67,28 @@ class CorpusThread extends Thread {
this
.
outFile
=
outfile
;
this
.
token
=
token
;
this
.
callbackUrl
=
callbackUrl
;
logger
.
info
(
"Input: "
+
this
.
inFile
+
" output: "
+
this
.
outFile
+
" functions: "
+
functions
+
" params: "
+
this
.
props
+
" token: "
+
this
.
token
+
" callback: "
+
this
.
callbackUrl
);
}
public
void
run
()
{
ArrayList
<
String
>
functionList
=
new
ArrayList
<>(
Arrays
.
asList
(
functionNames
.
split
(
","
)));
Set
<
String
>
allFunctions
=
CorpusServices
.
getCorpusFunctions
()
;
CorpusIO
cio
=
new
CorpusIO
();
report
.
addNote
(
"CorpusWebServices"
,
"Starting run at "
+
DateTimeFormatter
.
ofPattern
(
"yyyy/MM/dd HH:mm:ss"
).
format
(
LocalDateTime
.
now
()));
try
{
// Create corpus from given input file/folder
Corpus
corpus
=
new
Corpus
(
cio
.
read
(
new
File
(
inFile
).
toURI
().
toURL
(),
report
))
;
Corpus
corpus
;
if
(
corpusName
!=
null
&&
!
corpusName
.
isEmpty
())
{
URL
baseUrl
=
new
File
(
inFile
).
toURI
().
toURL
();
corpus
=
new
Corpus
(
corpusName
,
baseUrl
,
cio
.
read
(
baseUrl
,
report
))
;
}
else
corpus
=
new
Corpus
(
cio
.
read
(
new
File
(
inFile
).
toURI
().
toURL
(),
report
))
;
logger
.
info
(
"Loaded "
+
corpus
.
getCorpusData
().
size
()
+
" corpus files"
);
logger
.
info
(
"Got report: "
+
report
.
getFullReports
());
// For all functions to be applied, get their canonical name and create an object for them
Set
<
CorpusFunction
>
functions
=
new
HashSet
<>()
;
for
(
String
function
:
functionList
)
{
...
...
@@ -84,24 +103,24 @@ class CorpusThread extends Thread {
}
catch
(
IllegalArgumentException
|
NoSuchMethodException
|
InstantiationException
|
InvocationTargetException
|
IllegalAccessException
e
)
{
logger
.
warn
(
"Error creating {}"
,
canonical
);
report
.
addWarning
(
"CorpusService"
,
"Test "
+
function
+
" cannot be created"
);
report
.
addWarning
(
"Corpus
Web
Service
s
"
,
"Test "
+
function
+
" cannot be created"
);
// e.printStackTrace();
}
}
}
if
(!
found
)
{
// Warn if we could not find the function
report
.
addWarning
(
"CorpusService"
,
"Test "
+
function
+
" is not available"
);
report
.
addWarning
(
"Corpus
Web
Service
s
"
,
"Test "
+
function
+
" is not available"
);
logger
.
warn
(
"Function {} is not available in corpus services"
,
function
);
}
}
for
(
CorpusFunction
f
:
functions
)
{
logger
.
warn
(
"Running function {}"
,
f
.
getFunction
());
report
.
addNote
(
"CorpusService"
,
"Run test "
+
f
.
getFunction
());
report
.
addNote
(
"Corpus
Web
Service
s
"
,
"Run test "
+
f
.
getFunction
());
de
.
uni_hamburg
.
corpora
.
Report
result
=
f
.
execute
(
corpus
);
report
.
merge
(
result
);
report
.
addNote
(
"CorpusService"
,
"Finish test "
+
f
.
getFunction
());
report
.
addNote
(
"Corpus
Web
Service
s
"
,
"Finish test "
+
f
.
getFunction
());
logger
.
warn
(
"Done with function {}"
,
f
.
getFunction
());
}
}
catch
(
URISyntaxException
|
ClassNotFoundException
|
IOException
|
SAXException
|
JexmaraldaException
e
)
{
...
...
@@ -109,7 +128,8 @@ class CorpusThread extends Thread {
}
logger
.
info
(
"Done with all functions"
);
report
.
addNote
(
"CorpusService"
,
"Finished all tests"
);
report
.
addNote
(
"CorpusWebServices"
,
"Finished all tests at "
+
DateTimeFormatter
.
ofPattern
(
"yyyy/MM/dd HH:mm:ss"
).
format
(
LocalDateTime
.
now
()));
logger
.
info
(
"Creating report"
);
// Get summary
HashMap
<
ReportItem
.
Severity
,
Integer
>
summary
=
CorpusServices
.
generateSummary
(
report
);
...
...
@@ -181,7 +201,8 @@ public class CorpusChecker {
@GET
@Produces
(
MediaType
.
TEXT_PLAIN
)
public
Response
checkCorpus
(
@QueryParam
(
"input"
)
String
input
,
public
Response
checkCorpus
(
@QueryParam
(
"name"
)
String
name
,
@QueryParam
(
"input"
)
String
input
,
@QueryParam
(
"output"
)
String
output
,
@QueryParam
(
"functions"
)
String
functions
,
@QueryParam
(
"params"
)
String
paramStr
,
...
...
@@ -220,11 +241,25 @@ public class CorpusChecker {
return
Response
.
status
(
400
).
entity
(
"400 - "
+
errorMsg
).
build
();
}
Properties
params
=
new
Properties
();
if
(
paramStr
!=
null
)
{
if
(
paramStr
!=
null
&&
!
paramStr
.
equals
(
"{}"
)
)
{
ObjectMapper
mapper
=
new
ObjectMapper
();
params
.
putAll
(
mapper
.
convertValue
(
paramStr
,
Map
.
class
));
try
{
JsonNode
node
=
mapper
.
readTree
(
paramStr
);
// Copy the jason data into the properties
for
(
Iterator
<
Map
.
Entry
<
String
,
JsonNode
>>
it
=
node
.
fields
();
it
.
hasNext
();
)
{
Map
.
Entry
<
String
,
JsonNode
>
e
=
it
.
next
();
params
.
put
(
e
.
getKey
(),
e
.
getValue
().
textValue
());
}
logger
.
info
(
params
.
toString
());
//params.putAll(mapper.convertValue(paramStr,Map.class));
}
catch
(
JsonProcessingException
e
)
{
e
.
printStackTrace
();
}
}
else
{
params
=
new
Properties
();
}
CorpusThread
ct
=
new
CorpusThread
(
input
,
output
,
functions
,
params
,
token
,
callbackUrl
);
CorpusThread
ct
=
new
CorpusThread
(
name
,
input
,
output
,
functions
,
params
,
token
,
callbackUrl
);
ct
.
start
();
Main
.
addThread
(
ct
);
return
Response
.
ok
().
entity
(
"Executing "
+
functions
+
" on "
+
input
+
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment