Feature Idea (with own code adaptation): Adding datasets into lyse dataframe instead of just attributes

[Cardoran]

We are using and manipulating arrays in single-shot scripts and experienced errors when saving these into the h5-file attributes. We did this because the actual datasets didn't show up in the lyse dataframe and loading from disk is slow. We recently implemented this such that datasets are loaded into the dataframe and we now work with arrays without a problem. Our code changes (copied from gitlab) are below. Was this excluded deliberately and/or does this cause any other issues we don't run into in our experiment?
Best regards
Cardoran

From 224780030e490abecc57bdf5de18a560dac6e1a0 Mon Sep 17 00:00:00 2001
Date: Mon, 28 Apr 2025 13:16:31 +0200
Subject: [PATCH] datasets are now added to and updated in the dataframe when
modified in a single shot script

---

lyse/init.py | 7 +++++++
lyse/dataframe_utilities.py | 6 ++++--
2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/lyse/init.py b/lyse/init.py
index cb4dc8a..b0cb4fe 100644
--- a/lyse/init.py
+++ b/lyse/init.py
@@ -752,6 +752,13 @@
class Run(object):
self.h5_file[group].create_dataset(name, data=data, **kwargs)
for key, val in attrs.items():
self.h5_file[group][name].attrs[key] = val
+
+ if spinning_top:
+ if self.h5_path not in _updated_data:
+ _updated_data[self.h5_path] = {}
+ if group.startswith('results'):
+ toplevel = group.replace('results/', '', 1)
+ _updated_data[self.h5_path][toplevel, name] = data

@open_file('r')
def get_traces(self, *names,):
diff --git a/lyse/dataframe_utilities.py b/lyse/dataframe_utilities.py
index 728a783..a8b6883 100644
--- a/lyse/dataframe_utilities.py
+++ b/lyse/dataframe_utilities.py
@@ -15,7 +15,7 @@
import pandas
import tzlocal
import labscript_utils.shared_drive
from labscript_utils.connections import _ensure_str
-from labscript_utils.properties import get_attributes
+from labscript_utils.properties import get_attributes,get_group_items
from labscript_utils.shot_utils import get_shot_globals

@@ -31,7 +31,9 @@
def get_nested_dict_from_shot(filepath):
if 'results' in h5_file:
for groupname in h5_file['results']:
resultsgroup = h5_file['results'][groupname]
- row[groupname] = get_attributes(resultsgroup)
+ new_row = get_attributes(resultsgroup)
+ new_row.update(get_group_items(resultsgroup))
+ row[groupname] = new_row
if 'images' in h5_file:
for orientation in h5_file['images'].keys():
if isinstance(h5_file['images'][orientation], h5py.Group):

--
GitLab

From 07297fcb38780a82d6483eb4ccdd1c8df4eac358 Mon Sep 17 00:00:00 2001
Date: Fri, 30 May 2025 14:16:39 +0200
Subject: [PATCH] circumvent an error with writing arrays into the results.
will post a nan into the first row though but without errors

---

lyse/main.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lyse/main.py b/lyse/main.py
index 2db7358..c2a4279 100644
--- a/lyse/main.py
+++ b/lyse/main.py
@@ -1413,7 +1413,10 @@ class DataFrameModel(QtCore.QObject):
# 'object'
self.dataframe[column_name] = self.dataframe[column_name].astype('object')
# Now that the column exists and has dtype object, we can set the value:
- self.dataframe.at[row_number, column_name] = value
+ try:
+ self.dataframe.at[row_number, column_name] = value
+ except ValueError:
+ self.dataframe.at[row_number,column_name] = None #set none-element first for column-creation

dataframe_already_updated = True

--
GitLab