Skip to content
Snippets Groups Projects
Commit af78f39b authored by Nicola Tarocco's avatar Nicola Tarocco Committed by Nicola
Browse files

xml, ipynb, csv: refactor

parent 0456da35
No related branches found
No related tags found
No related merge requests found
Showing
with 185 additions and 507 deletions
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
import d3 from "d3";
import flight from "flightjs";
function CSV_D3JS() {
var CSV_D3JS;
this.tabulate = function(data, target, columns) {
var table = d3
.select(target)
.append("table")
.classed({
table: true,
"table-hover": true,
"table-bordered": true,
});
CSV_D3JS.thead = table.append("thead");
CSV_D3JS.tbody = table.append("tbody");
CSV_D3JS.columns = columns;
CSV_D3JS.data = data;
// append the header row
CSV_D3JS.thead
.append("tr")
.selectAll("th")
.data(CSV_D3JS.columns)
.enter()
.append("th")
.text(function(column) {
return column;
});
CSV_D3JS.next = 1;
CSV_D3JS.chunk_size = 500;
CSV_D3JS.chunks = Math.ceil(CSV_D3JS.data.length / CSV_D3JS.chunk_size);
CSV_D3JS.loadNext(undefined, {
id: CSV_D3JS.id,
});
if (CSV_D3JS.chunks > 1) {
CSV_D3JS.trigger(document, "showLoader", {
id: CSV_D3JS.id,
});
}
return true;
};
this.loadNext = function(ev, data) {
if (data.id === CSV_D3JS.id && CSV_D3JS.next <= CSV_D3JS.chunks) {
// create a row for each object in the data chunk
CSV_D3JS.tbody
.selectAll("tr")
.data(CSV_D3JS.data.slice(0, CSV_D3JS.next * CSV_D3JS.chunk_size))
.enter()
.append("tr")
.selectAll("td")
.data(function(row) {
return CSV_D3JS.columns.map(function(column) {
return { column: column, value: row[column] };
});
})
.enter()
.append("td")
.text(function(d) {
return d.value;
});
if (CSV_D3JS.next === CSV_D3JS.chunks) {
CSV_D3JS.trigger(document, "hideLoader", {
id: CSV_D3JS.id,
});
}
CSV_D3JS.next += 1;
}
};
this.after("initialize", function() {
CSV_D3JS = this;
CSV_D3JS.id = CSV_D3JS.node.id;
var delimiter = CSV_D3JS.$node.data("csv-delimiter"),
encoding = CSV_D3JS.$node.data("csv-encoding"),
resource = CSV_D3JS.$node.data("csv-source"),
dsv = d3.dsv(delimiter, "text/csv; charset=" + encoding);
dsv(resource, function(data) {
var col = Object.keys(data[0]);
CSV_D3JS.tabulate(data, CSV_D3JS.node, col);
});
this.on(document, "loadNext", this.loadNext);
});
}
export default flight.component(CSV_D3JS);
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
* Copyright (C) 2015-2023 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
import $ from 'jquery';
import CSV_D3JS from './csv_d3js';
import Loader from './loader.js';
$(function () {
$("[data-csv-source]").each(function () {
CSV_D3JS.attachTo($(this));
})
$("[data-csv-target]").each(function () {
Loader.attachTo($(this));
})
})
import $ from "jquery";
import Papa from "papaparse";
(function ($, Papa) {
function createDataElement(htmlTag, innerText) {
const node = document.createElement(htmlTag);
const textNode = document.createTextNode(innerText);
node.appendChild(textNode);
return node;
}
const URL = $("#app").attr("data-csv-source");
const maxRowsPerChunk = 50;
const $tableHeader = $("#table-header");
const $tableBody = $("#table-body");
const $showMore = $("#show-more");
let isFirst = true;
let currentStep = 0;
let papaParser = null;
$showMore.hide(); // hide it when init
$showMore.on("click", function () {
currentStep = 0;
papaParser.resume();
});
Papa.parse(URL, {
download: true,
skipEmptyLines: true,
step: function (results, parser) {
papaParser = papaParser || parser;
currentStep++;
console.debug("CSV previewer: rendering step", results);
const row = results.data || [];
const tableRowEl = document.createElement("tr");
let tableColEl;
if (isFirst) {
$tableHeader.append(tableRowEl);
tableColEl = "th";
isFirst = false;
} else {
$tableBody.append(tableRowEl);
tableColEl = "td";
}
row.forEach((col) => {
const node = createDataElement(tableColEl, col);
tableRowEl.appendChild(node);
});
if (currentStep >= maxRowsPerChunk) {
parser.pause();
$showMore.show();
}
},
error: function (err, file, inputElem, reason) {
console.error(
"CSV previewer: error rendering CSV file: ",
err,
file,
inputElem,
reason
);
},
complete: function (results, file) {
console.debug("CSV previewer: rendering completed", results, file);
$showMore.hide();
},
});
})($, Papa);
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
import flight from "flightjs";
function Loader() {
var Loader;
this.handleShowLoader = function(ev, data) {
if (data.id === Loader.id) {
this.$node.show();
}
};
this.handleHideLoader = function(ev, data) {
if (data.id === Loader.id) {
this.$node.hide();
}
};
this.after("initialize", function() {
Loader = this;
Loader.id = Loader.$node.data("csv-target");
Loader.on(document, "showLoader", Loader.handleShowLoader);
Loader.on(document, "hideLoader", Loader.handleHideLoader);
Loader.on("click", function(ev) {
ev.preventDefault();
Loader.trigger(document, "loadNext", {
id: Loader.id,
});
});
Loader.$node.hide();
});
}
export default flight.component(Loader);
/*
* This file is part of Invenio.
* Copyright (C) 2015-2020 CERN.
* Copyright (C) 2015-2023 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
import $ from "jquery";
import Papa from 'papaparse';
import Papa from "papaparse";
function createDataElement(htmlTag, innerText, parentNode) {
let node = document.createElement(htmlTag);
let textnode = document.createTextNode(innerText);
node.appendChild(textnode);
parentNode.appendChild(node);
(function ($, Papa) {
function createDataElement(htmlTag, innerText) {
const node = document.createElement(htmlTag);
const textNode = document.createTextNode(innerText);
node.appendChild(textNode);
return node;
}
var link = "https://" + window.location.host + $("[data-csv-source]").attr('data-csv-source');
var header = true;
var limit = 50;
var step = 0;
var global_parser ;
$("#more_data").click(function () {
step = 0;
global_parser.resume();
});
const URL = $("#app").attr("data-csv-source");
const maxRowsPerChunk = 50;
const $tableHeader = $("#table-header");
const $tableBody = $("#table-body");
const $showMore = $("#show-more");
let isFirst = true;
let currentStep = 0;
let papaParser = null;
$showMore.hide(); // hide it when init
$showMore.on("click", function () {
currentStep = 0;
papaParser.resume();
});
var results = Papa.parse(link, {
Papa.parse(URL, {
download: true,
skipEmptyLines: true,
step: function(row, parser) {
var element;
var node = document.createElement("tr");
if (header) {
header = false;
element ="th";
$("#tableHeader").append(node);
step: function (results, parser) {
papaParser = papaParser || parser;
currentStep++;
console.debug("CSV previewer: rendering step", results);
const row = results.data || [];
const tableRowEl = document.createElement("tr");
let tableColEl;
if (isFirst) {
$tableHeader.append(tableRowEl);
tableColEl = "th";
isFirst = false;
} else {
element = "td";
$("#tableBody").append(node);
$tableBody.append(tableRowEl);
tableColEl = "td";
}
for(let j = 0; j < row.data.length; j++) {
createDataElement(element, row.data[j], node);
}
step++;
if (step >= limit) {
row.forEach((col) => {
const node = createDataElement(tableColEl, col);
tableRowEl.appendChild(node);
});
if (currentStep >= maxRowsPerChunk) {
parser.pause();
global_parser = parser;
$showMore.show();
}
},
complete: function(results) {
$("#more_data").hide();
}
error: function (err, file, inputElem, reason) {
console.error(
"CSV previewer: error rendering CSV file: ",
err,
file,
inputElem,
reason
);
},
complete: function (results, file) {
console.debug("CSV previewer: rendering completed", results, file);
$showMore.hide();
},
});
})($, Papa);
......@@ -7,5 +7,3 @@
# under the terms of the MIT License; see LICENSE file for more details.
"""Implementations of different viewers."""
from __future__ import absolute_import, print_function
......@@ -9,7 +9,7 @@
"""Jupyter Notebook previewer."""
from __future__ import absolute_import, unicode_literals
import os
import nbformat
from flask import render_template
......@@ -20,6 +20,12 @@ from ..proxies import current_previewer
previewable_extensions = ["ipynb"]
# relative paths to the extra CSS of the default `lab` theme
NBCONVERT_THEME_LAB_EXTRA_CSS = [
os.path.join("nbconvert", "templates", "lab", "static", "index.css"),
os.path.join("nbconvert", "templates", "lab", "static", "theme-light.css"),
]
def render(file):
"""Generate the result HTML."""
......@@ -28,11 +34,11 @@ def render(file):
try:
notebook = nbformat.reads(content.decode("utf-8"), as_version=4)
except nbformat.reader.NotJSONError:
return _("Error: Not a json file"), {}
return _("Error: Not a ipynb/json file"), {}
html_exporter = HTMLExporter(template="lab", embed_images=True)
html_exporter = HTMLExporter(embed_images=True, sanitize_html=True)
html_exporter.template_file = "base"
(body, resources) = html_exporter.from_notebook_node(notebook)
body, resources = html_exporter.from_notebook_node(notebook)
return body, resources
......@@ -44,10 +50,14 @@ def can_preview(file):
def preview(file):
"""Render the IPython Notebook."""
body, resources = render(file)
if "inlining" in resources:
default_jupyter_nb_style = resources["inlining"]["css"][0]
else:
default_jupyter_nb_style = ""
if "inlining" in resources:
default_jupyter_nb_style += resources["inlining"]["css"][0]
# the include_css func will load extra CSS from disk
if "include_css" in resources:
fn = resources["include_css"]
for extra_css_path in NBCONVERT_THEME_LAB_EXTRA_CSS:
default_jupyter_nb_style += fn(extra_css_path)
return render_template(
"invenio_previewer/ipynb.html",
file=file,
......
......@@ -8,8 +8,6 @@
"""Markdown rendering using mistune library."""
from __future__ import absolute_import, unicode_literals
import bleach
import mistune
from flask import render_template
......
/*
* This file is part of Invenio.
* Copyright (C) 2016-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
code[class*="language-"],
pre[class*="language-"] {
line-height: 1.0;
}
:not(pre) > code[class*="language-"],
pre[class*="language-"] {
background: white;
}
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
'use strict';
define(function(require) {
var $ = require('node_modules/jquery/dist/jquery'),
d3 = require('node_modules/d3/d3');
return require('node_modules/flightjs/build/flight').component(CSV_D3JS);
function CSV_D3JS() {
var CSV_D3JS;
this.tabulate = function (data, target, columns) {
var table = d3.select(target).append("table").classed({
'table': true,
'table-hover': true,
'table-bordered': true,
});
CSV_D3JS.thead = table.append("thead");
CSV_D3JS.tbody = table.append("tbody");
CSV_D3JS.columns = columns;
CSV_D3JS.data = data;
// append the header row
CSV_D3JS.thead.append("tr")
.selectAll("th")
.data(CSV_D3JS.columns)
.enter()
.append("th")
.text(function(column) { return column; });
CSV_D3JS.next = 1;
CSV_D3JS.chunk_size = 500;
CSV_D3JS.chunks = Math.ceil(CSV_D3JS.data.length/CSV_D3JS.chunk_size);
CSV_D3JS.loadNext(undefined, {
id: CSV_D3JS.id
});
if (CSV_D3JS.chunks > 1) {
CSV_D3JS.trigger(document, "showLoader", {
id: CSV_D3JS.id
});
}
return true;
}
this.loadNext = function (ev, data) {
if (data.id===CSV_D3JS.id && CSV_D3JS.next <= CSV_D3JS.chunks) {
// create a row for each object in the data chunk
var rows = CSV_D3JS.tbody.selectAll("tr")
.data(CSV_D3JS.data.slice(0,CSV_D3JS.next*CSV_D3JS.chunk_size))
.enter()
.append("tr");
// create a cell in each row for each column
var cells = rows.selectAll("td")
.data(function(row) {
return CSV_D3JS.columns.map(function(column) {
return {column: column, value: row[column]};
});
})
.enter()
.append("td")
.text(function(d) { return d.value; });
if (CSV_D3JS.next === CSV_D3JS.chunks) {
CSV_D3JS.trigger(document, "hideLoader", {
id: CSV_D3JS.id
});
}
CSV_D3JS.next+=1;
}
}
this.after('initialize', function () {
CSV_D3JS = this;
CSV_D3JS.id = CSV_D3JS.node.id;
var delimiter = CSV_D3JS.$node.data("csv-delimiter"),
encoding = CSV_D3JS.$node.data("csv-encoding"),
resource = CSV_D3JS.$node.data("csv-source"),
dsv = d3.dsv(delimiter, "text/csv; charset=" + encoding);
dsv(resource, function(data) {
var col = Object.keys(data[0]);
CSV_D3JS.tabulate(data, CSV_D3JS.node, col);
});
this.on(document, 'loadNext', this.loadNext)
});
}
});
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
requirejs(['js/csv_previewer/csv_d3js', 'js/csv_previewer/loader'], function(CSV_D3JS, Loader) {
$(function () {
$("[data-csv-source]").each(function () {
CSV_D3JS.attachTo($(this));
});
$("[data-csv-target]").each(function () {
Loader.attachTo($(this));
});
});
})
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
'use strict';
define(function(require) {
var $ = require('node_modules/jquery/dist/jquery'),
d3 = require('node_modules/d3/d3');
return require('node_modules/flightjs/build/flight').component(Loader);
function Loader() {
var Loader;
this.handleShowLoader = function (ev, data) {
if (data.id === Loader.id){
this.$node.show();
}
};
this.handleHideLoader = function (ev, data) {
if (data.id === Loader.id){
this.$node.hide();
}
};
this.after('initialize', function () {
Loader = this;
Loader.id = Loader.$node.data('csv-target');
Loader.on(document, 'showLoader', Loader.handleShowLoader);
Loader.on(document, 'hideLoader', Loader.handleHideLoader);
Loader.on('click', function (ev) {
ev.preventDefault();
Loader.trigger(document, 'loadNext', {
id: Loader.id
});
});
Loader.$node.hide();
});
}
});
/*
* This file is part of Invenio.
* Copyright (C) 2015-2019 CERN.
*
* Invenio is free software; you can redistribute it and/or modify it
* under the terms of the MIT License; see LICENSE file for more details.
*/
var f = parent.document.getElementById('preview-iframe');
if (f) {
var handleFullScreenClick = (function () {
var isFullScreen = false;
var pos = f.style.position,
zIndex = f.style.zIndex,
height = f.style.height,
width = f.style.width,
top = f.style.top,
left = f.style.left,
backgroundColor = f.style.backgroundColor;
return function () {
if (isFullScreen) {
isFullScreen = false;
f.style.position = pos;
f.style.zIndex = zIndex;
f.style.height = height;
f.style.width = width;
f.style.top = top;
f.style.left = left;
f.style.backgroundColor = backgroundColor;
parent.document.body.style.overflow = "";
} else {
isFullScreen = true;
f.style.position = "fixed";
f.style.zIndex = 9999;
f.style.height = "100%";
f.style.width = "100%";
f.style.top = 0;
f.style.left = 0;
f.style.backgroundColor="white";
parent.document.body.style.overflow = "hidden";
}
};
}());
var fsbtn = f.contentDocument.getElementById('fullScreenMode');
var secfsbtn = f.contentDocument.getElementById('secondaryFullScreenMode');
if (fsbtn) fsbtn.addEventListener('click', handleFullScreenClick);
if (secfsbtn) secfsbtn.addEventListener('click', handleFullScreenClick);
} else {
var fsbtn = document.getElementById('fullScreenMode');
var secfsbtn = document.getElementById('secondaryFullScreenMode');
if (fsbtn) fsbtn.remove();
if (secfsbtn) secfsbtn.remove();
}
{# -*- coding: utf-8 -*-
This file is part of Invenio.
Copyright (C) 2015-2019 CERN.
Copyright (C) 2015-2023 CERN.
Invenio is free software; you can redistribute it and/or modify it
under the terms of the MIT License; see LICENSE file for more details.
#}
{#- D3 view for viewing csv files -#}
{%- extends config.PREVIEWER_ABSTRACT_TEMPLATE %}
{% block panel %}
<div id="d3_csv_previewer" data-csv-source="{{ file.uri }}"
data-csv-delimiter="{{ delimiter }}" data-csv-encoding="{{ encoding }}"></div>
<a data-csv-target="d3_csv_previewer" href="">{{ _('Show more') }}</a>
<div id="app" data-csv-source="{{ file.uri }}">
<table class="ui selectable celled table unstackable">
<thead id="table-header">
</thead>
<tbody id="table-body">
</tbody>
</table>
<button id="show-more" class="btn" >{{_('Show more')}}</button>
</div>
{% endblock %}
{# -*- coding: utf-8 -*-
This file is part of Invenio.
Copyright (C) 2015-2019 CERN.
Copyright (C) 2015-2020 CERN.
Invenio is free software; you can redistribute it and/or modify it
under the terms of the MIT License; see LICENSE file for more details.
......@@ -17,11 +17,5 @@
{% endblock %}
{% block panel %}
<div class="container">
<div class="row">
<div class="col-md-12">
{{ content | sanitize_html() | safe }}
</div>
</div>
</div>
{{ content | safe }}
{% endblock %}
{# -*- coding: utf-8 -*-
This file is part of Invenio.
Copyright (C) 2015-2020 CERN.
Copyright (C) 2015-2023 CERN.
Invenio is free software; you can redistribute it and/or modify it
under the terms of the MIT License; see LICENSE file for more details.
#}
{#- D3 view for viewing csv files -#}
{%- extends config.PREVIEWER_ABSTRACT_TEMPLATE %}
{% block panel %}
<div id="app" data-csv-source="{{ file.uri }}">
<table class="ui selectable celled table unstackable">
<thead id="tableHeader">
<thead id="table-header">
</thead>
<tbody id="tableBody">
<tbody id="table-body">
</tbody>
</table>
<button id="more_data" class="ui button" >{{_('Show more')}}</button>
<button id="show-more" class="ui button m-10" >{{_('Show more')}}</button>
</div>
{% endblock %}
......@@ -14,10 +14,8 @@
<style>
{{ inline_style | safe }}
</style>
{{ webpack["nbconvert_index_css.css"] }}
{{ webpack["nbconvert_theme_light_css.css"] }}
{% endblock %}
{% block panel %}
{{ content | sanitize_html() | safe }}
{{ content | safe }}
{% endblock %}
......@@ -25,17 +25,8 @@
"""JS/CSS bundles for Previewer."""
from os import path
# This seems like a hack... there must be a better way of doing it
import nbconvert
from invenio_assets.webpack import WebpackThemeBundle
nbconvert_path = (
path.dirname(nbconvert.__file__)
+ "/../../../../share/jupyter/nbconvert/templates/lab/static/"
)
previewer = WebpackThemeBundle(
__name__,
"assets",
......@@ -81,8 +72,6 @@ previewer = WebpackThemeBundle(
"bottom_css": "./scss/invenio_previewer/bottom.scss",
"simple_image_css": "./scss/invenio_previewer/simple_image.scss",
"txt_css": "./scss/invenio_previewer/txt.scss",
"nbconvert_index_css": nbconvert_path + "index.css",
"nbconvert_theme_light_css": nbconvert_path + "theme-light.css",
},
dependencies={
"flightjs": "~1.5.1",
......
......@@ -34,12 +34,9 @@ install_requires =
invenio-i18n>=1.3.1
invenio-pidstore>=1.2.3
invenio-records-ui>=1.2.0
ipython>=4.1.0
mistune>=0.8.1
nbconvert>=6.0,<7.0
nbclient>=0.5,<1.0
nbconvert>=7,<8
nbformat>=5.1,<6.0
tornado>=6.1,<7.0
[options.extras_require]
tests =
......
......@@ -192,7 +192,7 @@ def test_ipynb_extension(testapp, webassets, record):
"cell_type": "markdown",
"metadata": {},
"source": [
"This is an example notebook."
"This is an example notebook.<script>alert();</script>"
]
}
],
......@@ -223,7 +223,10 @@ def test_ipynb_extension(testapp, webassets, record):
with testapp.test_client() as client:
res = client.get(preview_url(record["control_number"], "test.ipynb"))
assert "This is an example notebook." in res.get_data(as_text=True)
as_text = res.get_data(as_text=True)
assert "This is an example notebook." in as_text
# test HTML tag sanitize
assert "<script>alert();</script>" not in as_text
def test_simple_image_extension(testapp, webassets, record):
......@@ -232,8 +235,9 @@ def test_simple_image_extension(testapp, webassets, record):
with testapp.test_client() as client:
res = client.get(preview_url(record["control_number"], "test.png"))
assert '<img src="' in res.get_data(as_text=True)
assert 'class="previewer-simple-image"' in res.get_data(as_text=True)
as_text = res.get_data(as_text=True)
assert '<img src="' in as_text
assert 'class="previewer-simple-image"' in as_text
def test_txt_extension_valid_file(testapp, webassets, record):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment