Deploy 2021-1.0

azure-pipelines.yml

@@ -3,7 +3,7 @@ resources:
     - repository: templates
       type: github
       name: NSLS-II/profile-collection-ci
-      ref: refs/heads/collection-2020-2.0rc7-1
+      ref: refs/heads/collection-2021-1.0
       endpoint: github
 
 jobs:

startup/00-base.py

@@ -1,5 +1,45 @@
 print(f"Loading {__file__}...")
 
+###############################################################################
+# TODO: remove this block once https://github.com/bluesky/ophyd/pull/959 is
+# merged/released.
+from datetime import datetime
+from ophyd.signal import EpicsSignalBase, EpicsSignal, DEFAULT_CONNECTION_TIMEOUT
+
+def print_now():
+    return datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S.%f')
+
+def wait_for_connection_base(self, timeout=DEFAULT_CONNECTION_TIMEOUT):
+    '''Wait for the underlying signals to initialize or connect'''
+    if timeout is DEFAULT_CONNECTION_TIMEOUT:
+        timeout = self.connection_timeout
+    # print(f'{print_now()}: waiting for {self.name} to connect within {timeout:.4f} s...')
+    start = time.time()
+    try:
+        self._ensure_connected(self._read_pv, timeout=timeout)
+        # print(f'{print_now()}: waited for {self.name} to connect for {time.time() - start:.4f} s.')
+    except TimeoutError:
+        if self._destroyed:
+            raise DestroyedError('Signal has been destroyed')
+        raise
+
+def wait_for_connection(self, timeout=DEFAULT_CONNECTION_TIMEOUT):
+    '''Wait for the underlying signals to initialize or connect'''
+    if timeout is DEFAULT_CONNECTION_TIMEOUT:
+        timeout = self.connection_timeout
+    # print(f'{print_now()}: waiting for {self.name} to connect within {timeout:.4f} s...')
+    start = time.time()
+    self._ensure_connected(self._read_pv, self._write_pv, timeout=timeout)
+    # print(f'{print_now()}: waited for {self.name} to connect for {time.time() - start:.4f} s.')
+
+EpicsSignalBase.wait_for_connection = wait_for_connection_base
+EpicsSignal.wait_for_connection = wait_for_connection
+###############################################################################
+
+from ophyd.signal import EpicsSignalBase
+# EpicsSignalBase.set_default_timeout(timeout=10, connection_timeout=10)  # old style
+EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10)  # new style
+
 import nslsii
 import matplotlib as mpl
 from IPython.terminal.prompts import Prompts, Token
@@ -35,12 +75,6 @@ def in_prompt_tokens(self, cli=None):
 # Disable BestEffortCallback to plot ring current
 bec.disable_plots()
 
-
-# Temporary fix before it's fixed in ophyd
-import logging
-logger = logging.getLogger('ophyd')
-logger.setLevel('WARNING')
-
 from pathlib import Path
 
 import appdirs
@@ -54,15 +88,49 @@ def in_prompt_tokens(self, cli=None):
     import zict
 
     class PersistentDict(zict.Func):
+        """
+        A MutableMapping which syncs it contents to disk.
+        The contents are stored as msgpack-serialized files, with one file per item
+        in the mapping.
+        Note that when an item is *mutated* it is not immediately synced:
+        >>> d['sample'] = {"color": "red"}  # immediately synced
+        >>> d['sample']['shape'] = 'bar'  # not immediately synced
+        but that the full contents are synced to disk when the PersistentDict
+        instance is garbage collected.
+        """
         def __init__(self, directory):
             self._directory = directory
             self._file = zict.File(directory)
+            self._cache = {}
             super().__init__(self._dump, self._load, self._file)
+            self.reload()
+
+            # Similar to flush() or _do_update(), but without reference to self
+            # to avoid circular reference preventing collection.
+            # NOTE: This still doesn't guarantee call on delete or gc.collect()!
+            #       Explicitly call flush() if immediate write to disk required.
+            def finalize(zfile, cache, dump):
+                zfile.update((k, dump(v)) for k, v in cache.items())
+
+            import weakref
+            self._finalizer = weakref.finalize(
+                self, finalize, self._file, self._cache, PersistentDict._dump)
 
         @property
         def directory(self):
             return self._directory
 
+        def __setitem__(self, key, value):
+            self._cache[key] = value
+            super().__setitem__(key, value)
+
+        def __getitem__(self, key):
+            return self._cache[key]
+
+        def __delitem__(self, key):
+            del self._cache[key]
+            super().__delitem__(key)
+
         def __repr__(self):
             return f"<{self.__class__.__name__} {dict(self)!r}>"
 
@@ -83,6 +151,16 @@ def _load(file):
                 object_hook=msgpack_numpy.decode,
                 raw=False)
 
+        def flush(self):
+            """Force a write of the current state to disk"""
+            for k, v in self.items():
+                super().__setitem__(k, v)
+
+        def reload(self):
+            """Force a reload from disk, overwriting current cache"""
+            self._cache = dict(super().items())
+
+
 # runengine_metadata_dir = appdirs.user_data_dir(appname="bluesky") / Path("runengine-metadata")
 runengine_metadata_dir = Path('/nsls2/xf05id1/shared/config/runengine-metadata')
 

startup/01-liveplot-workaround.py

@@ -5,6 +5,11 @@
 from bluesky.callbacks.core import CallbackBase, get_obj_fields, make_class_safe
 from ophyd.sim import det1, det2, motor
 
+
+import logging
+logger = logging.getLogger('bluesky')
+
+
 class HackLivePlot(LivePlot):
     """
     Build a function that updates a plot from a stream of Events.

startup/21-cameras.py

@@ -12,38 +12,47 @@
 from ophyd.areadetector.cam import AreaDetectorCam
 from ophyd.device import Component as Cpt
 
+from ophyd.areadetector.plugins import (ImagePlugin_V33, TIFFPlugin_V33,
+                                        ROIPlugin_V33, StatsPlugin_V33)
+
 
 # BPM Camera
-class SRXTIFFPlugin(TIFFPlugin,
+class SRXTIFFPlugin(TIFFPlugin_V33,
                     FileStoreTIFF,
                     FileStoreIterativeWrite):
-    file_number_sync = None
+    ...
+
+
+class SRXAreaDetectorCam(AreaDetectorCam):
+    pool_max_buffers = None
 
 
 class BPMCam(SingleTrigger, AreaDetector):
-    cam = Cpt(AreaDetectorCam, 'cam1:')
-    image_plugin = Cpt(ImagePlugin, 'image1:')
+    cam = Cpt(SRXAreaDetectorCam, 'cam1:')
+    image_plugin = Cpt(ImagePlugin_V33, 'image1:')
 
     tiff = Cpt(SRXTIFFPlugin, 'TIFF1:',
                write_path_template='/epicsdata/bpm1-cam1/2020/11/07/')
                #write_path_template='/epicsdata/bpm1-cam1/%Y/%m/%d/',
                #root='/epicsdata', reg=db.reg)
                # write_path_template='/nsls2/xf05id1/data/bpm1-cam1/%Y/%m/%d/',
                # root='/nsls2/xf05id1')
-    roi1 = Cpt(ROIPlugin, 'ROI1:')
-    roi2 = Cpt(ROIPlugin, 'ROI2:')
-    roi3 = Cpt(ROIPlugin, 'ROI3:')
-    roi4 = Cpt(ROIPlugin, 'ROI4:')
-    stats1 = Cpt(StatsPlugin, 'Stats1:')
-    stats2 = Cpt(StatsPlugin, 'Stats2:')
-    stats3 = Cpt(StatsPlugin, 'Stats3:')
-    stats4 = Cpt(StatsPlugin, 'Stats4:')
+    roi1 = Cpt(ROIPlugin_V33, 'ROI1:')
+    roi2 = Cpt(ROIPlugin_V33, 'ROI2:')
+    roi3 = Cpt(ROIPlugin_V33, 'ROI3:')
+    roi4 = Cpt(ROIPlugin_V33, 'ROI4:')
+    stats1 = Cpt(StatsPlugin_V33, 'Stats1:')
+    stats2 = Cpt(StatsPlugin_V33, 'Stats2:')
+    stats3 = Cpt(StatsPlugin_V33, 'Stats3:')
+    stats4 = Cpt(StatsPlugin_V33, 'Stats4:')
     # this is flakey?
-    # stats5 = C(StatsPlugin, 'Stats5:')
+    # stats5 = Cpt(StatsPlugin_V33, 'Stats5:')
 
 
 bpmAD = BPMCam('XF:05IDA-BI:1{BPM:1-Cam:1}', name='bpmAD', read_attrs=[])
-bpmAD.read_attrs = ['stats1', 'stats2', 'stats3', 'stats4']
+bpmAD.wait_for_connection()
+
+bpmAD.read_attrs = ['stats1', 'stats2', 'stats3', 'stats4', 'tiff']
 bpmAD.stats1.read_attrs = ['total']
 bpmAD.stats2.read_attrs = ['total']
 bpmAD.stats3.read_attrs = ['total']

startup/31-xspress3.py

@@ -12,7 +12,7 @@
 from ophyd.device import Staged
 from enum import Enum
 
-from hxntools.detectors.xspress3 import (
+from nslsii.detectors.xspress3 import (
     XspressTrigger,
     Xspress3Detector,
     Xspress3Channel,

startup/35-dexela.py

@@ -7,7 +7,7 @@
 
 import ophyd
 from hxntools.detectors.dexela import (DexelaDetector,)
-from hxntools.detectors.xspress3 import (logger, )
+from nslsii.detectors.xspress3 import (logger, )
 from databroker.assets.handlers import HandlerBase
 from ophyd.areadetector.filestore_mixins import (FileStoreIterativeWrite,
                                                  FileStoreHDF5IterativeWrite,

startup/92-interactive.py

@@ -10,7 +10,9 @@
 
 
 from bluesky.log import config_bluesky_logging
-config_bluesky_logging(file='/var/log/bluesky/bluesky_ipython.log', level='DEBUG')
+bluesky_ipython_log = os.getenv('BLUESKY_IPYTHON_LOG_FILE',
+                                os.path.expanduser('~/.cache/bluesky/log/bluesky_ipython.log'))
+config_bluesky_logging(file=bluesky_ipython_log, level='DEBUG')
 
 def plot_crab(hdr):
     fig, ax = plt.subplots()