Merge branch 'develop' into 'master'

Develop

See merge request las-nq/openqlab!45

.coveragerc

@@ -2,7 +2,7 @@
 omit = */OldImporters/*
 
 [report]
-fail_under = 88.6
+fail_under = 89.3
 
 [html]
 directory = htmlcov

.editorconfig

@@ -37,4 +37,4 @@ indent_style = space
 indent_size = 2
 
 [settings]
-known_third_party = docutils,jsonpickle,matplotlib,numpy,openqlab,pandas,pipenv,pkg_resources,recommonmark,scipy,serial,setuptools,sphinx,sphinx_rtd_theme,tabulate
+known_third_party = docutils,jsonpickle,matplotlib,mock_visa,numpy,openqlab,pandas,pipenv,pkg_resources,pyvisa,recommonmark,scipy,serial,setuptools,sphinx,sphinx_rtd_theme,tabulate

Pipfile

@@ -43,6 +43,8 @@ six = ">=1.12.0"
 python-dateutil = ">=2.7.5"
 eml-parser = ">=1.11"
 DateTime = "*"
+pyvisa = "*"
+pyvisa-py = "*"
 docutils = "==0.15.2"
 
 [pipenv]

README.md

@@ -93,4 +93,4 @@ pre-commit install
 ```
 
 ----
-(c) 2019, LasNQ @ Uni Hamburg
+(c) 2020, LasNQ @ Uni Hamburg

doc/conf.py

@@ -2,6 +2,7 @@
 #
 import re
 import sys
+from datetime import datetime
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -75,9 +76,9 @@ source_suffix = [".rst", ".md"]
 master_doc = "index"
 
 # General information about the project.
-project = u"openqlab"
-copyright = u"2017, LasNQ @ Uni Hamburg"
-author = u"LasNQ @ Uni Hamburg"
+project = "openqlab"
+copyright = f"{datetime.now().year}, LasNQ @ Uni Hamburg"
+author = "LasNQ @ Uni Hamburg"
 
 
 # The short X.Y version.
@@ -130,7 +131,7 @@ htmlhelp_basename = "openqlabdoc"
 
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
+latex_elements = {  # type: ignore
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
@@ -149,14 +150,14 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "openqlab.tex", u"openqlab Documentation", u"LasNQ group", "manual")
+    (master_doc, "openqlab.tex", "openqlab Documentation", "LasNQ group", "manual")
 ]
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [(master_doc, "openqlab.tex", u"openqlab Documentation", [author], 1)]
+man_pages = [(master_doc, "openqlab.tex", "openqlab Documentation", [author], 1)]
 
 # -- Options for Texinfo output -------------------------------------------
 
@@ -167,7 +168,7 @@ texinfo_documents = [
     (
         master_doc,
         "openqlab.tex",
-        u"openqlab Documentation",
+        "openqlab Documentation",
         author,
         "openqlab",
         "Open source lab toolbox for quantum optical experiments.",

doc/conversion.rst

+:mod:`openqlab.conversion` -- Data Conversions
+**********************************************
+
+dB
+--
+
+Decibel conversions.
+
+.. automodule:: openqlab.conversion.db
+    :members:
+
+
+Utils
+-----
+
+Some unit utils.
+
+.. automodule:: openqlab.conversion.utils
+    :members:
+
+
+Wavelength
+----------
+
+Wavelength calucations.
+
+.. automodule:: openqlab.conversion.wavelength
+    :members:

doc/index.rst

@@ -24,29 +24,21 @@ Reference
    :maxdepth: 2
 
    analysis
+   conversion
    io
    plots
    datacontainer
 
-
-License
-=======
-
-Note that although this package might become free software at some point, it
-currently isn't because it is unclear who actually *owns* the software.
-Therefore, usage is only allowed within the Las-NQ group at the University of
-Hamburg, unless expressly stated otherwise.
-
 Contribute
 ==========
 
 Most of the original content in this package was written during the PhD theses of
 Sebastian Steinlechner and Tobias Gehring. It is currently maintained by
-Sebastian Steinlechner, Christian Darsow-Fromm and is looking for more
+Christian Darsow-Fromm and Jan Petermann and is looking for more
 volunteers who would like to contribute.
 
-If you want to contribute, feel free to do so. The source code is hosted on the
-gitlab server at AEI Hannover, https://gitlab.aei.uni-hannover.de/las-nq/lab/.
+If you want to contribute, feel free to do so. The source code is hosted on
+https://gitlab.rrz.uni-hamburg.de/las-nq/openqlab
 
 Indices and tables
 ==================

doc/usage.rst

@@ -48,10 +48,12 @@ automatically recognized, run
 Next Steps
 ----------
 
-To work with measurement data and do further analysis, here is some further reading for you:
+To work with measurement data and do further analysis, here is some further
+reading for you:
 
 .. toctree::
     :maxdepth: 2
 
     Working with DataFrames/DataContainer <dataframes>
     Designing a Servo <servodesign>
+    Visa Importer <visa_importer>

doc/visa_importer.md

+# Visa Importer
+
+Importing data directly from network attached device works via the `io.read` method.
+
+Currently there are the following devices implemented:
+
+- Keysight oscilloscope (LAN)
+- HP4395a (GPIB)
+
+Feel free to add more devices.
+
+```python
+In  [0]: from openqlab import io
+
+In  [1]: data = io.read("TCPIP::hostname_or_ip::INSTR")
+
+In  [2]: data.head()
+Out [2]:
+------------------------------------------------------------
+xincrement : 8.064e-07
+xreference : 0
+average_count : 1
+xorigin : 0.00754
+yUnit : V
+points : 992
+type : normal
+xUnit : s
+------------------------------------------------------------                                                        
+                 1         2        3         4
+Time
+0.007540  0.000252 -0.004599 -1.09422  0.029460
+0.007541 -0.000150 -0.003795 -1.09422 -0.010741
+0.007542  0.000654 -0.003393 -1.09422 -0.010741
+0.007542  0.000654 -0.003393 -1.29523  0.009359
+0.007543  0.000654 -0.004197 -1.09422  0.009359
+```
+
+**Do not forget to save the data!**
+
+```python
+dc.to_csv("filename.csv")
+```

src/openqlab/io/__init__.py

@@ -124,15 +124,15 @@ def read(
     else:
         selected_importers = BaseImporter.auto_importers()
 
+    data: List[DataContainer] = [
+        _import(data_file, selected_importers, **kwargs) for data_file in files_list
+    ]
+
     if append is True:
         axis = 0
     else:
         axis = 1
 
-    data: List[DataContainer] = [
-        _import(data_file, selected_importers, **kwargs) for data_file in files_list
-    ]
-
     if as_list:
         return data
     return DataContainer.concat(data, axis=axis)

src/openqlab/io/base_importer.py

@@ -19,6 +19,8 @@ from typing import (
     cast,
 )
 
+import pyvisa
+
 from openqlab.io.data_container import DataContainer
 from openqlab.io.importers import utils
 
@@ -59,7 +61,42 @@ class BaseImporter(ABC):
 
 
 class VisaImporter(BaseImporter, ABC):
-    pass
+    """VisaImporter template."""
+
+    IDN_STARTS_WITH = cast(str, abstract_class_attribute)
+    IDN_STARTS_WITH.__doc__ = "Start of the device IDN"
+    ADDRESS_STARTS_WITH = cast(str, abstract_class_attribute)
+    ADDRESS_STARTS_WITH.__doc__ = "Mandatory start of the address"
+
+    def __init__(self, data: Union[str, IO, Path], inst=None):
+        if not isinstance(data, str):
+            raise utils.UnknownFileType(f"{self.NAME}: not a string")
+        if not data.lower().startswith(self.ADDRESS_STARTS_WITH):
+            raise utils.UnknownFileType(f"{self.NAME}: not a Visa address")
+
+        rm = pyvisa.ResourceManager("@py")
+        if inst is None:
+            self._inst = rm.open_resource(data)
+        else:
+            self._inst = inst
+
+        self._check_connection()
+
+    @property
+    def idn(self) -> str:
+        return self.query("*IDN?").strip()
+
+    def query(self, query: str) -> str:
+        return self._inst.query(query)
+
+    def write(self, command: str):
+        self._inst.write(command)
+
+    def _check_connection(self):
+        try:
+            assert self.idn.startswith(self.IDN_STARTS_WITH)
+        except (AssertionError, pyvisa.errors.VisaIOError):
+            raise utils.UnknownFileType(f"{self.NAME}: cannot open connection")
 
 
 class StreamImporter(BaseImporter, ABC):

src/openqlab/io/importers/hp4395a_gpib.py

@@ -38,9 +38,11 @@ class HP4395A_GPIB(VisaImporter):
 
     NAME = "HP4395A_GPIB"
     AUTOIMPORTER = False
+    ADDRESS_STARTS_WITH = ""
 
     def __init__(self, file):
         self.file = file
+        super().__init__()
 
     def read(self):
         serial_port, gpib_address = self.file.split("::")

src/openqlab/io/importers/keysight_visa.py

+from typing import List
+
+import numpy as np
+
+from openqlab.io.base_importer import VisaImporter
+from openqlab.io.data_container import DataContainer
+from openqlab.io.importers import utils
+
+
+class KeysightVisa(VisaImporter):
+    NAME = "KeysightVisa"
+    AUTOIMPORTER = True
+    IDN_STARTS_WITH: str = "KEYSIGHT TECHNOLOGIES,DSO-X"
+    ADDRESS_STARTS_WITH: str = "tcpip::"
+    MAX_COLUMNS = 4
+    NUMBER_OF_POINTS = 1000
+
+    def read(self):
+        data = self._read_data()
+        output = DataContainer.concat(data, axis=1)
+        output.index.name = "Time"
+        output.header = self._header
+
+        return output
+
+    def _read_data(self) -> List[np.ndarray]:
+        self.write(":WAVeform:POINTs:MODE NORMal")
+        self.write(f":WAVeform:POINts {self.NUMBER_OF_POINTS}")
+        self.write(":WAVeform:FORMat ASCII")
+
+        self.write(":STOP")
+
+        self._read_meta_data()
+
+        xorigin = self._header["xorigin"]
+        step = self._header["xincrement"]
+        points = self._header["points"]
+        # Using arange this way gives always the correct number of points
+        self._index = np.arange(points) * step + xorigin
+
+        data = []
+        for i in range(1, 1 + self.MAX_COLUMNS):
+            channel_active = self.query(f":CHANnel{i}:DISPlay?").strip()
+            if channel_active == "1":
+                data.append(DataContainer({i: self._read_column(i)}, index=self._index))
+
+        if not data:
+            raise utils.ImportFailed(
+                f"'{self.NAME}' importer: No active trace on the scope"
+            )
+
+        # TODO start only if stopped
+        self.write(":RUN")
+        return data
+
+    def _read_meta_data(self):
+        preamble = self.query("WAV:PREamble?").strip()
+        entries = preamble.split(",")
+
+        type_dict = {
+            0: "normal",
+            1: "peak detect",
+            2: "average",
+            3: "hresolution",
+        }
+
+        self._header = dict(
+            # format=entries[0],
+            type=type_dict[int(entries[1])],
+            points=int(entries[2]),
+            average_count=int(entries[3]),
+            xincrement=float(entries[4]),
+            xorigin=float(entries[5]),
+            xreference=int(entries[6]),
+            xUnit="s",
+            yUnit="V",
+            # yincrement=float(entries[7]),
+            # yorigin=float(entries[8]),
+            # yreference=int(entries[9]),
+        )
+
+    def _read_column(self, channel: int) -> np.ndarray:
+        """
+        The data looks like this:
+
+        #800000139 6.43216e-003, 9.24623e-003, 4.02010e-003, 1.28643e-002
+
+        The first digit (8) defines the number of digits for the following
+        number (00000139) which is the length of the data.
+        """
+        self.write(f":WAVeform:SOURce CHANnel{channel}")
+        raw_data = self.query("WAV:DATA?").strip()
+
+        if not raw_data or not raw_data[0] == "#":
+            raise utils.ImportFailed(f"{self.NAME}: The data does not start with #")
+
+        try:
+            n = int(raw_data[1])
+            n_digits = int(raw_data[2 : n + 2])
+
+            clipped_data = raw_data[n + 2 :]
+            assert (
+                len(clipped_data) == n_digits
+            ), f"len data: {len(clipped_data)}, n_digits: {n_digits}"
+
+            data = np.array(clipped_data.split(","), dtype=float)
+        except (ValueError, AssertionError):
+            raise utils.ImportFailed(f"{self.NAME}: Could not process the data")
+
+        return data

src/tests/test_io/test_importers/test_keysight_visa/mock_visa.py

+from re import match
+from typing import List
+
+
+class MockVisa:
+    def __init__(self):
+        self.log = []
+        self.read_termination = ""
+        self.waveform_channel: str = None
+        self.waveform_points: int = None
+        self.waveform_format: str = None
+        self.channels_enabled: List[str] = ["0"] * 4
+        self.channel_data = {
+            "channel1": "#800000139 1.35683e-002,-1.19603e-002,-3.11608e-003, 6.33216e-003, 9.14623e-003, 7.13618e-003, 1.03523e-002, 3.11608e-003, 5.93015e-003, 1.19603e-002",
+            "channel2": "#800000139 2.35683e-002,-2.19603e-002,-4.11608e-003, 7.33216e-003, 9.14623e-003, 7.13618e-003, 1.03523e-002, 3.11608e-003, 5.93015e-003, 2.19603e-002",
+            "channel3": "#800000139 3.35683e-002,-3.19603e-002,-5.11608e-003, 6.33216e-003, 9.14623e-003, 7.13618e-003, 1.03523e-002, 3.11608e-003, 5.93015e-003, 3.19603e-002",
+            "channel4": "#800000139 4.35683e-002,-4.19603e-002,-7.11608e-003, 6.33216e-003, 9.14623e-003, 7.13618e-003, 1.03523e-002, 3.11608e-003, 5.93015e-003, 4.19603e-002",
+        }
+        self.idn = "KEYSIGHT TECHNOLOGIES,DSO-X 3024T,MY57452230,07.30.2019051434"
+
+    def open_resource(self, _: str):
+        return self
+
+    def write(self, command: str):
+        known_commands = [":run", ":stop", ":waveform:points:mode"]
+
+        self.log.append(f"write: {command}")
+        print(self.log)
+
+        commands: list = command.lower().strip().split(" ")
+        if commands[0] == ":waveform:source":
+            self.waveform_channel = commands[1]
+        elif commands[0] == ":waveform:points":
+            self.waveform_points = int(commands[1])
+        elif commands[0] == ":waveform:format":
+            self.waveform_points = commands[1]
+        elif commands[0] in known_commands:
+            pass
+        else:
+            raise ValueError(f"Unknown command: {command}")
+
+    def query(self, query: str):
+        self.log.append(f"query: {query}")
+        print(self.log)
+
+        query = query.lower().strip()
+
+        if query == "wav:preamble?":
+            return "+4,+0,+10,+1,+8.00000000E-005,-4.00000000E-004,+0,+1.57035200E-006,-1.00000000E-004,+32768"
+        if query == "wav:data?":
+            return self.channel_data[self.waveform_channel]
+        if query == "*idn?":
+            return self.idn
+        m = match(r":channel(.):display\?", query)
+        if m is not None:
+            chan = int(m.group(1))
+            return self.channels_enabled[chan - 1]
+
+        raise ValueError(f"Unknown query: {query}")

src/tests/test_io/test_importers/test_keysight_visa/test_keysight_visa.py

+import unittest
+from pathlib import Path
+
+from mock_visa import MockVisa
+from openqlab.io.importers.keysight_visa import KeysightVisa
+from openqlab.io.importers.utils import ImportFailed, UnknownFileType
+
+filedir = Path(__file__).parent
+
+
+class TestKeysightVisa(unittest.TestCase):
+    def setUp(self):
+        mock = MockVisa()
+        self.importer = KeysightVisa("TCPIP::mockaddress", inst=mock)
+
+    def test_idn(self):
+        self.assertEqual(
+            self.importer.idn,
+            "KEYSIGHT TECHNOLOGIES,DSO-X 3024T,MY57452230,07.30.2019051434",
+        )
+
+    def test_read_data(self):
+        self.importer._inst.channels_enabled = ["1"] * 4
+        dc = self.importer.read()
+        self.assertEqual(len(dc.columns), 4)
+        self.assertEqual(dc.index.name, "Time")
+
+    def test_index(self):
+        self.importer._inst.channels_enabled = ["1"] * 4
+        dc = self.importer.read()
+        self.assertAlmostEqual(dc.index[5], 0)
+
+    def test_missing_data(self):
+        self.importer._inst.channels_enabled[0] = "1"
+        self.importer._inst.channel_data["channel1"] = ""
+        with self.assertRaises(ImportFailed):
+            dc = self.importer.read()
+
+    def test_wrong_data(self):
+        self.importer._inst.channels_enabled[0] = "1"
+        self.importer._inst.channel_data["channel1"] = "031243"
+        with self.assertRaises(ImportFailed):
+            dc = self.importer.read()
+
+    def test_without_active_trace(self):
+        with self.assertRaises(ImportFailed):
+            dc = self.importer.read()
+
+    def test_no_number_on_second_place(self):
+        self.importer._inst.channels_enabled[1] = "1"
+        self.importer._inst.channel_data["channel2"] = "#x3284023"
+        with self.assertRaises(ImportFailed):
+            dc = self.importer.read()
+
+    def test_clipped_data(self):
+        self.importer._inst.channels_enabled[0] = "1"
+        self.importer._inst.channel_data[
+            "channel1"
+        ] = "#800000140 1.35683e-002,-1.19603e-002,-3.11608e-003, 6.33216e-003, 9.14623e-003, 7.13618e-003, 1.03523e-002, 3.11608e-003, 5.93015e-003, 1.19603e-002"
+        with self.assertRaises(ImportFailed):
+            dc = self.importer.read()
+
+    def test_data_values(self):
+        self.importer._inst.channels_enabled = ["1"] * 4
+        dc = self.importer.read()
+        print(dc)
+        self.assertEqual(dc[1].iloc[0], 1.35683e-2)
+        self.assertEqual(dc[2].iloc[0], 2.35683e-2)
+        self.assertEqual(dc[3].iloc[0], 3.35683e-2)
+        self.assertEqual(dc[4].iloc[0], 4.35683e-2)
+
+        self.assertEqual(dc[1].iloc[1], -1.19603e-2)
+        self.assertEqual(dc[2].iloc[1], -2.19603e-2)
+        self.assertEqual(dc[3].iloc[1], -3.19603e-2)
+        self.assertEqual(dc[4].iloc[1], -4.19603e-2)
+
+        self.assertEqual(dc[1].iloc[-1], 1.19603e-2)
+        self.assertEqual(dc[2].iloc[-1], 2.19603e-2)
+        self.assertEqual(dc[3].iloc[-1], 3.19603e-2)
+        self.assertEqual(dc[4].iloc[-1], 4.19603e-2)
+
+    def test_open_real_resource(self):
+        with self.assertRaises(ConnectionRefusedError):
+            self.importer = KeysightVisa("TCPIP::localhost::INSTR")
+
+    def test_wrong_device_idn(self):
+        mock = MockVisa()
+        importer = KeysightVisa("TCPIP::mockaddress", inst=mock)
+        importer._inst.idn = "Something wrong"
+        with self.assertRaises(UnknownFileType):
+            importer._check_connection()