diff --git a/docs/api.rst b/docs/api.rst index d58bf3de783b4e5dbf01243a711cc52b7c69e1bc..def7907125cb252446bc783ec4ca3271172e94d9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -50,10 +50,10 @@ Webpack Previewers ---------- -CSV (d3.js) -~~~~~~~~~~~ +CSV (Papaparser) +~~~~~~~~~~~~~~~~ -.. automodule:: invenio_previewer.extensions.csv_dthreejs +.. automodule:: invenio_previewer.extensions.csv_papaparsejs :members: :undoc-members: diff --git a/invenio_previewer/__init__.py b/invenio_previewer/__init__.py index b875e33e33da911e1f6ea66d500df0a7d37e71c5..67c33dc06d3e872de40e5e1f3ea492f7879bf477 100644 --- a/invenio_previewer/__init__.py +++ b/invenio_previewer/__init__.py @@ -304,7 +304,7 @@ Now define the priority for all previewers by adding the newly created >>> PREVIEWER_PREVIEWERS_ORDER = [ ... 'invenio_previewer.extensions.txt_previewer', -... 'invenio_previewer.extensions.csv_dthreejs', +... 'invenio_previewer.extensions.csv_papaparsejs', ... 'invenio_previewer.extensions.json_prismjs', ... 'invenio_previewer.extensions.xml_prismjs', ... 'invenio_previewer.extensions.simple_image', diff --git a/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/csv_d3js.js b/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/csv_d3js.js deleted file mode 100644 index 57e8c5cb31ae7f71196305c7264fbf48f952f748..0000000000000000000000000000000000000000 --- a/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/csv_d3js.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * This file is part of Invenio. - * Copyright (C) 2015-2020 CERN. - * - * Invenio is free software; you can redistribute it and/or modify it - * under the terms of the MIT License; see LICENSE file for more details. - */ - -import d3 from "d3"; -import flight from "flightjs"; - -function CSV_D3JS() { - var CSV_D3JS; - - this.tabulate = function(data, target, columns) { - var table = d3 - .select(target) - .append("table") - .classed({ - table: true, - "table-hover": true, - "table-bordered": true, - }); - CSV_D3JS.thead = table.append("thead"); - CSV_D3JS.tbody = table.append("tbody"); - CSV_D3JS.columns = columns; - CSV_D3JS.data = data; - - // append the header row - CSV_D3JS.thead - .append("tr") - .selectAll("th") - .data(CSV_D3JS.columns) - .enter() - .append("th") - .text(function(column) { - return column; - }); - - CSV_D3JS.next = 1; - CSV_D3JS.chunk_size = 500; - CSV_D3JS.chunks = Math.ceil(CSV_D3JS.data.length / CSV_D3JS.chunk_size); - - CSV_D3JS.loadNext(undefined, { - id: CSV_D3JS.id, - }); - if (CSV_D3JS.chunks > 1) { - CSV_D3JS.trigger(document, "showLoader", { - id: CSV_D3JS.id, - }); - } - - return true; - }; - - this.loadNext = function(ev, data) { - if (data.id === CSV_D3JS.id && CSV_D3JS.next <= CSV_D3JS.chunks) { - // create a row for each object in the data chunk - CSV_D3JS.tbody - .selectAll("tr") - .data(CSV_D3JS.data.slice(0, CSV_D3JS.next * CSV_D3JS.chunk_size)) - .enter() - .append("tr") - .selectAll("td") - .data(function(row) { - return CSV_D3JS.columns.map(function(column) { - return { column: column, value: row[column] }; - }); - }) - .enter() - .append("td") - .text(function(d) { - return d.value; - }); - - if (CSV_D3JS.next === CSV_D3JS.chunks) { - CSV_D3JS.trigger(document, "hideLoader", { - id: CSV_D3JS.id, - }); - } - CSV_D3JS.next += 1; - } - }; - - this.after("initialize", function() { - CSV_D3JS = this; - CSV_D3JS.id = CSV_D3JS.node.id; - - var delimiter = CSV_D3JS.$node.data("csv-delimiter"), - encoding = CSV_D3JS.$node.data("csv-encoding"), - resource = CSV_D3JS.$node.data("csv-source"), - dsv = d3.dsv(delimiter, "text/csv; charset=" + encoding); - - dsv(resource, function(data) { - var col = Object.keys(data[0]); - CSV_D3JS.tabulate(data, CSV_D3JS.node, col); - }); - - this.on(document, "loadNext", this.loadNext); - }); -} - -export default flight.component(CSV_D3JS); diff --git a/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/init.js b/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/init.js index 189ad4d2c2923e83b3e64c371f3c815b7b37db13..b84e99020aeb2f12573e0100a685902b5350b866 100644 --- a/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/init.js +++ b/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/init.js @@ -7,14 +7,53 @@ */ import $ from "jquery"; -import CSV_D3JS from "./csv_d3js"; -import Loader from "./loader.js"; - -$(function () { - $("[data-csv-source]").each(function () { - CSV_D3JS.attachTo($(this)); - }); - $("[data-csv-target]").each(function () { - Loader.attachTo($(this)); - }); +import Papa from 'papaparse'; + +function createDataElement(htmlTag, innerText, parentNode) { + let node = document.createElement(htmlTag); + let textnode = document.createTextNode(innerText); + node.appendChild(textnode); + parentNode.appendChild(node); +} + +var link = "https://" + window.location.host + $("[data-csv-source]").attr('data-csv-source'); +var header = true; +var limit = 50; +var step = 0; +var global_parser ; +$("#more_data").click(function () { + step = 0; + global_parser.resume(); +}); + + +var results = Papa.parse(link, { + download: true, + skipEmptyLines: true, + step: function(row, parser) { + var element; + var node = document.createElement("tr"); + if (header) { + header = false; + element ="th"; + $("#tableHeader").append(node); + } else { + element = "td"; + $("#tableBody").append(node); + } + + for(let j = 0; j < row.data.length; j++) { + createDataElement(element, row.data[j], node); + } + step++; + if (step >= limit) { + parser.pause(); + global_parser = parser; + } + }, + complete: function(results) { + $("#more_data").hide(); + } + }); + diff --git a/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/loader.js b/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/loader.js deleted file mode 100644 index a7cddac893f7bab44f259ec0ce159ac707215214..0000000000000000000000000000000000000000 --- a/invenio_previewer/assets/semantic-ui/js/invenio_previewer/csv_previewer/loader.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of Invenio. - * Copyright (C) 2015-2020 CERN. - * - * Invenio is free software; you can redistribute it and/or modify it - * under the terms of the MIT License; see LICENSE file for more details. - */ - -import flight from "flightjs"; - -function Loader() { - var Loader; - - this.handleShowLoader = function (ev, data) { - if (data.id === Loader.id) { - this.$node.show(); - } - }; - - this.handleHideLoader = function (ev, data) { - if (data.id === Loader.id) { - this.$node.hide(); - } - }; - - this.after("initialize", function () { - Loader = this; - Loader.id = Loader.$node.data("csv-target"); - - Loader.on(document, "showLoader", Loader.handleShowLoader); - Loader.on(document, "hideLoader", Loader.handleHideLoader); - Loader.on("click", function (ev) { - ev.preventDefault(); - Loader.trigger(document, "loadNext", { - id: Loader.id, - }); - }); - - Loader.$node.hide(); - }); -} - -export default flight.component(Loader); diff --git a/invenio_previewer/config.py b/invenio_previewer/config.py index 8e473530974df7d2c8aeb917cce0322ff63c554a..f68748ffdfd37da03a457649d8caaa006e18a4b9 100644 --- a/invenio_previewer/config.py +++ b/invenio_previewer/config.py @@ -33,7 +33,7 @@ PREVIEWER_ZIP_MAX_FILES = 1000 """Max number of files showed in the ZIP previewer.""" PREVIEWER_PREFERENCE = [ - "csv_dthreejs", + "csv_papaparsejs", "simple_image", "json_prismjs", "xml_prismjs", diff --git a/invenio_previewer/extensions/csv_dthreejs.py b/invenio_previewer/extensions/csv_dthreejs.py deleted file mode 100644 index 76396c3fee98d4090b39209aaad10a86a80fbb8e..0000000000000000000000000000000000000000 --- a/invenio_previewer/extensions/csv_dthreejs.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of Invenio. -# Copyright (C) 2015-2019 CERN. -# -# Invenio is free software; you can redistribute it and/or modify it -# under the terms of the MIT License; see LICENSE file for more details. - -"""Render a CSV file using d3.js.""" - -import csv - -from flask import current_app, render_template - -from ..proxies import current_previewer -from ..utils import detect_encoding - -previewable_extensions = ["csv", "dsv"] - - -def validate_csv(file): - """Return dialect information about given csv file.""" - max_file_size = current_app.config.get( - "PREVIEWER_MAX_FILE_SIZE_BYTES", 10 * 1024 * 1024 - ) - if file.size > max_file_size: - return False - - try: - # Detect encoding and dialect - with file.open() as fp: - encoding = detect_encoding(fp, default="utf-8") - sample = fp.read( - current_app.config.get("PREVIEWER_CSV_VALIDATION_BYTES", 1024) - ) - allowed_delimiters = current_app.config.get( - "PREVIEWER_CSV_SNIFFER_ALLOWED_DELIMITERS", None - ) - delimiter = ( - csv.Sniffer() - .sniff(sample=sample.decode(encoding), delimiters=allowed_delimiters) - .delimiter - ) - is_valid = True - except Exception as e: - current_app.logger.debug("File {0} is not valid CSV: {1}".format(file.uri, e)) - encoding = "" - delimiter = "" - is_valid = False - - return {"delimiter": delimiter, "encoding": encoding, "is_valid": is_valid} - - -def can_preview(file): - """Determine if the given file can be previewed.""" - if file.is_local() and file.has_extensions(".csv", ".dsv"): - return validate_csv(file)["is_valid"] - return False - - -def preview(file): - """Render the appropriate template with embed flag.""" - file_info = validate_csv(file) - return render_template( - "invenio_previewer/csv_bar.html", - file=file, - delimiter=file_info["delimiter"], - encoding=file_info["encoding"], - js_bundles=current_previewer.js_bundles + ["d3_csv.js"], - css_bundles=current_previewer.css_bundles, - ) diff --git a/invenio_previewer/extensions/csv_papaparsejs.py b/invenio_previewer/extensions/csv_papaparsejs.py new file mode 100644 index 0000000000000000000000000000000000000000..21a5100b5ff6d847aea70904f39bd96426e487ec --- /dev/null +++ b/invenio_previewer/extensions/csv_papaparsejs.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2015-2019 CERN. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Render a CSV file using Papaparse.""" + +from flask import current_app, render_template + +from ..proxies import current_previewer + +previewable_extensions = ["csv", "dsv"] + + +def can_preview(file): + """Determine if the given file can be previewed.""" + if file.is_local() and file.has_extensions(".csv", ".dsv"): + return True + return False + + +def preview(file): + """Render the appropriate template with embed flag.""" + return render_template( + "invenio_previewer/csv_bar.html", + file=file, + js_bundles=current_previewer.js_bundles + ["papaparse_csv.js"], + css_bundles=current_previewer.css_bundles, + ) diff --git a/invenio_previewer/extensions/ipynb.py b/invenio_previewer/extensions/ipynb.py index 5401a8d17d05a0fb87f81a489ef5e2896b606be1..06cd5880a427edc8a09293c301be7d773ee9edcc 100644 --- a/invenio_previewer/extensions/ipynb.py +++ b/invenio_previewer/extensions/ipynb.py @@ -13,6 +13,7 @@ from __future__ import absolute_import, unicode_literals import nbformat from flask import render_template +from invenio_i18n import gettext as _ from nbconvert import HTMLExporter from ..proxies import current_previewer @@ -24,10 +25,12 @@ def render(file): """Generate the result HTML.""" with file.open() as fp: content = fp.read() + try: + notebook = nbformat.reads(content.decode("utf-8"), as_version=4) + except nbformat.reader.NotJSONError: + return _("Error: Not a json file"), {} - notebook = nbformat.reads(content.decode("utf-8"), as_version=4) - - html_exporter = HTMLExporter() + html_exporter = HTMLExporter(template="lab", embed_images=True) html_exporter.template_file = "base" (body, resources) = html_exporter.from_notebook_node(notebook) return body, resources @@ -41,7 +44,10 @@ def can_preview(file): def preview(file): """Render the IPython Notebook.""" body, resources = render(file) - default_jupyter_nb_style = resources["inlining"]["css"][0] + if "inlining" in resources: + default_jupyter_nb_style = resources["inlining"]["css"][0] + else: + default_jupyter_nb_style = "" return render_template( "invenio_previewer/ipynb.html", file=file, diff --git a/invenio_previewer/extensions/xml_prismjs.py b/invenio_previewer/extensions/xml_prismjs.py index c11810b8a1de36bae4a5504ebf352d2cea6fd209..a98a3c775be3204378db2318c1871a2e002aae34 100644 --- a/invenio_previewer/extensions/xml_prismjs.py +++ b/invenio_previewer/extensions/xml_prismjs.py @@ -11,6 +11,7 @@ import xml.dom.minidom from flask import current_app, render_template +from invenio_i18n import gettext as _ from ..proxies import current_previewer from ..utils import detect_encoding @@ -22,9 +23,14 @@ def render(file): """Pretty print the XML file for rendering.""" with file.open() as fp: encoding = detect_encoding(fp, default="utf-8") - file_content = fp.read().decode(encoding) - parsed_xml = xml.dom.minidom.parseString(file_content) - return parsed_xml.toprettyxml(indent=" ", newl="") + try: + file_content = fp.read().decode(encoding) + parsed_xml = xml.dom.minidom.parseString(file_content) + return parsed_xml.toprettyxml(indent=" ", newl="") + except UnicodeDecodeError: + return _( + "Error decoding the file. Are you sure it is '{encoding}'?" + ).format(encoding) def validate_xml(file): diff --git a/invenio_previewer/templates/semantic-ui/invenio_previewer/csv_bar.html b/invenio_previewer/templates/semantic-ui/invenio_previewer/csv_bar.html index c4a9703a6ce35d58a776e9ce45d1a95ef308d516..e7e62e331899bccd944dcc54c1f714112a4f7090 100644 --- a/invenio_previewer/templates/semantic-ui/invenio_previewer/csv_bar.html +++ b/invenio_previewer/templates/semantic-ui/invenio_previewer/csv_bar.html @@ -12,7 +12,14 @@ {%- extends config.PREVIEWER_ABSTRACT_TEMPLATE %} {% block panel %} - <div id="d3_csv_previewer" data-csv-source="{{ file.uri }}" - data-csv-delimiter="{{ delimiter }}" data-csv-encoding="{{ encoding }}"></div> - <a data-csv-target="d3_csv_previewer" href="">{{ _('Show more') }}</a> +<div id="app" data-csv-source="{{ file.uri }}"> + + <table class="ui selectable celled table unstackable"> + <thead id="tableHeader"> + </thead> + <tbody id="tableBody"> + </tbody> + </table> + <button id="more_data" class="ui button" >{{_('Show more')}}</button> +</div> {% endblock %} diff --git a/invenio_previewer/templates/semantic-ui/invenio_previewer/ipynb.html b/invenio_previewer/templates/semantic-ui/invenio_previewer/ipynb.html index f60579d8dd451b6a28597b4a015a1e849167a9cf..335ffe4e8a6db6680ac1e8e835b08e50a4284fe3 100644 --- a/invenio_previewer/templates/semantic-ui/invenio_previewer/ipynb.html +++ b/invenio_previewer/templates/semantic-ui/invenio_previewer/ipynb.html @@ -14,14 +14,10 @@ <style> {{ inline_style | safe }} </style> +{{ webpack["nbconvert_index_css.css"] }} +{{ webpack["nbconvert_theme_light_css.css"] }} {% endblock %} {% block panel %} -<div class="ui container"> - <div class="ui padded grid column"> - <div class="column"> {{ content | sanitize_html() | safe }} - </div> - </div> -</div> {% endblock %} diff --git a/invenio_previewer/webpack.py b/invenio_previewer/webpack.py index 0b40e4a1e2f450a6441e4917bb9f7e7c34d88cd1..de48dc36a017dc4e453dd6c9da2c10877a2cbef8 100644 --- a/invenio_previewer/webpack.py +++ b/invenio_previewer/webpack.py @@ -25,8 +25,17 @@ """JS/CSS bundles for Previewer.""" +from os import path + +# This seems like a hack... there must be a better way of doing it +import nbconvert from invenio_assets.webpack import WebpackThemeBundle +nbconvert_path = ( + path.dirname(nbconvert.__file__) + + "/../../../../share/jupyter/nbconvert/templates/lab/static/" +) + previewer = WebpackThemeBundle( __name__, "assets", @@ -34,7 +43,7 @@ previewer = WebpackThemeBundle( themes={ "bootstrap3": dict( entry={ - "d3_csv": "./js/invenio_previewer/csv_previewer/init.js", + "papaparse_csv": "./js/invenio_previewer/csv_previewer/init.js", "previewer_theme": "./js/invenio_previewer/previewer_theme.js", "fullscreen_js": "./js/invenio_previewer/fullscreen.js", "prism_js": "./js/invenio_previewer/prismjs.js", @@ -46,7 +55,7 @@ previewer = WebpackThemeBundle( }, dependencies={ "bootstrap-sass": "~3.3.5", - "d3": "^3.5.17", + "papaparse": "^5.4.1", "flightjs": "~1.5.1", "font-awesome": "~4.5.0", "jquery": "^3.3.1", @@ -59,7 +68,7 @@ previewer = WebpackThemeBundle( ), "semantic-ui": dict( entry={ - "d3_csv": "./js/invenio_previewer/csv_previewer/init.js", + "papaparse_csv": "./js/invenio_previewer/csv_previewer/init.js", "previewer_theme": "./js/invenio_previewer/previewer_theme.js", "fullscreen_js": "./js/invenio_previewer/fullscreen.js", "prism_js": "./js/invenio_previewer/prismjs.js", @@ -72,12 +81,14 @@ previewer = WebpackThemeBundle( "bottom_css": "./scss/invenio_previewer/bottom.scss", "simple_image_css": "./scss/invenio_previewer/simple_image.scss", "txt_css": "./scss/invenio_previewer/txt.scss", + "nbconvert_index_css": nbconvert_path + "index.css", + "nbconvert_theme_light_css": nbconvert_path + "theme-light.css", }, dependencies={ - "d3": "^3.5.17", "flightjs": "~1.5.1", "font-awesome": "~4.5.0", "jquery": "^3.3.1", + "papaparse": "^5.4.1", "pdfjs-dist": "^1.4.192", "prismjs": "^1.15.0", }, diff --git a/setup.cfg b/setup.cfg index 50df8b0f90e97d5b214f9193b3e4a2f4e6851ff4..ddaf1512a3cb9c153fe6a90f8922996d641cb1c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ invenio_i18n.translations = invenio_assets.webpack = invenio_previewer_theme = invenio_previewer.webpack:previewer invenio_previewer.previewers = - csv_dthreejs = invenio_previewer.extensions.csv_dthreejs + csv_papaparsejs = invenio_previewer.extensions.csv_papaparsejs json_prismjs = invenio_previewer.extensions.json_prismjs simple_image = invenio_previewer.extensions.simple_image xml_prismjs = invenio_previewer.extensions.xml_prismjs diff --git a/tests/test_macros.py b/tests/test_macros.py index f73c440cd4c054df7846cd3ebacf78cf2a0482e2..875ff3b1fd2a718f169796f557c8255c12ba9cb1 100644 --- a/tests/test_macros.py +++ b/tests/test_macros.py @@ -73,26 +73,20 @@ def test_pdf_extension(testapp, webassets, record): assert "pdf-file-uri" in res.get_data(as_text=True) -def test_csv_dthreejs_extension(testapp, webassets, record): +def test_csv_papaparsejs_extension(testapp, webassets, record): """Test view with csv files.""" create_file(record, "test.csv", BytesIO(b"A,B\n1,2")) with testapp.test_client() as client: res = client.get(preview_url(record["control_number"], "test.csv")) assert 'data-csv-source="' in res.get_data(as_text=True) - assert 'data-csv-delimiter=","' in res.get_data(as_text=True) - - with patch("csv.Sniffer", side_effect=Exception): - res = client.get(preview_url(record["control_number"], "test.csv")) - assert "we are unfortunately not" in res.get_data(as_text=True) -def test_csv_dthreejs_delimiter(testapp, webassets, record): +def test_csv_papaparsejs_delimiter(testapp, webassets, record): """Test view with csv files.""" create_file(record, "test.csv", BytesIO(b"A#B\n1#2")) with testapp.test_client() as client: res = client.get(preview_url(record["control_number"], "test.csv")) assert 'data-csv-source="' in res.get_data(as_text=True) - assert 'data-csv-delimiter="#"' in res.get_data(as_text=True) def test_zip_extension(testapp, webassets, record, zip_fp):