Use offline files for everything

aiapy/calibrate/prep.py

@@ -13,7 +13,7 @@
 from sunpy.util.decorators import add_common_docstring
 
 from aiapy.calibrate.transform import _rotation_function_names
-from aiapy.calibrate.util import _select_epoch_from_correction_table, get_correction_table
+from aiapy.calibrate.util import CALIBRATION_VERSION, _select_epoch_from_correction_table, get_correction_table
 from aiapy.util import AIApyUserWarning
 from aiapy.util.decorators import validate_channel
 
@@ -115,7 +115,7 @@ def register(smap, *, missing=None, order=3, method="scipy"):
     return newmap
 
 
-def correct_degradation(smap, *, correction_table=None, calibration_version=None):
+def correct_degradation(smap, *, correction_table=None, calibration_version=CALIBRATION_VERSION):
     """
     Apply time-dependent degradation correction to an AIA map.
 
@@ -149,6 +149,9 @@ def correct_degradation(smap, *, correction_table=None, calibration_version=None
     --------
     degradation
     """
+    # Fallback for back-compatibility
+    if calibration_version is None:
+        calibration_version = CALIBRATION_VERSION
     d = degradation(
         smap.wavelength,
         smap.date,
@@ -165,7 +168,7 @@ def degradation(
     obstime,
     *,
     correction_table=None,
-    calibration_version=None,
+    calibration_version=CALIBRATION_VERSION,
 ) -> u.dimensionless_unscaled:
     r"""
     Correction to account for time-dependent degradation of the instrument.
@@ -215,12 +218,15 @@ def degradation(
     aiapy.response.Channel.wavelength_response
     aiapy.response.Channel.eve_correction
     """
+    # Fallback for backwards compatibility
+    if calibration_version is None:
+        calibration_version = CALIBRATION_VERSION
     if obstime.shape == ():
         obstime = obstime.reshape((1,))
     ratio = np.zeros(obstime.shape)
     poly = np.zeros(obstime.shape)
     # Do this outside of the loop to avoid repeated queries
-    correction_table = get_correction_table(correction_table=correction_table)
+    correction_table = get_correction_table(correction_table=correction_table, calibration_version=calibration_version)
     for i, t in enumerate(obstime):
         table = _select_epoch_from_correction_table(channel, t, correction_table, version=calibration_version)
 

aiapy/calibrate/spikes.py

@@ -148,10 +148,14 @@ def fetch_spikes(smap, *, as_coords=False):
     series = r"aia.lev1_euv_12s"
     if smap.wavelength in (1600, 1700, 4500) * u.angstrom:
         series = r"aia.lev1_uv_24s"
-    file = drms.Client().query(
-        f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]',
-        seg="spikes",
-    )
+    try:
+        file = drms.Client().query(
+            f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]',
+            seg="spikes",
+        )
+    except Exception as e:
+        msg = f"Could not retrieve spikes for {smap.date} at {smap.wavelength}.\n" f"Error message: {e}"
+        raise OSError(msg) from e
     _, spikes = fits.open(f'http://jsoc.stanford.edu{file["spikes"][0]}')
     # Loaded as floats, but they are actually integers
     spikes = spikes.data.astype(np.int32)

aiapy/calibrate/tests/test_spikes.py

@@ -12,6 +12,8 @@
 from aiapy.calibrate import fetch_spikes, respike
 from aiapy.util import AIApyUserWarning
 
+pytestmark = [pytest.mark.xfail(reason="JSOC is down")]
+
 
 @pytest.fixture
 def despiked_map():

aiapy/calibrate/tests/test_util.py

@@ -92,6 +92,7 @@ def test_obstime_out_of_range() -> None:
 
 
 @pytest.mark.remote_data
+@pytest.mark.xfail(reason="JSOC is down")
 def test_pointing_table() -> None:
     expected_columns = ["T_START", "T_STOP"]
     for c in ["094", "171", "193", "211", "304", "335", "1600", "1700", "4500"]:

aiapy/calibrate/util.py

@@ -2,7 +2,6 @@
 Utilities for computing intensity corrections.
 """
 
-import os
 import pathlib
 import warnings
 from urllib.parse import urljoin
@@ -12,12 +11,14 @@
 
 import astropy.io.ascii
 import astropy.units as u
+from astropy.io import ascii as astropy_ascii
 from astropy.table import QTable
 from astropy.time import Time
 
 import drms
 from sunpy import log
-from sunpy.net import attrs, jsoc
+from sunpy.net import attrs as a
+from sunpy.net import jsoc
 
 from aiapy import _SSW_MIRRORS
 from aiapy.data._manager import manager
@@ -33,7 +34,7 @@
 # Most recent version number for error tables; increment as new versions become available
 ERROR_VERSION = 3
 # URLs and SHA-256 hashes for each version of the error tables
-URL_HASH = {
+URL_HASH_ERROR_TABLE = {
     2: (
         [urljoin(mirror, AIA_ERROR_FILE.format(2)) for mirror in _SSW_MIRRORS],
         "ac97ccc48057809723c27e3ef290c7d78ee35791d9054b2188baecfb5c290d0a",
@@ -43,9 +44,47 @@
         "66ff034923bb0fd1ad20e8f30c7d909e1a80745063957dd6010f81331acaf894",
     ),
 }
+URL_HASH_POINTING_TABLE = (
+    "https://aia.lmsal.com/public/master_aia_pointing3h.csv",
+    "a2c80fa0ea3453c62c91f51df045ae04b771d5cbb51c6495ed56de0da2a5482e",
+)
+URL_HASH_RESPONSE_TABLE = {
+    10: (
+        [urljoin(mirror, "sdo/aia/response/aia_V10_20201119_190000_response_table.txt") for mirror in _SSW_MIRRORS],
+        "0a3f2db39d05c44185f6fdeec928089fb55d1ce1e0a805145050c6356cbc6e98",
+    ),
+    9: (
+        [urljoin(mirror, "sdo/aia/response/aia_V9_20200706_215452_response_table.txt") for mirror in _SSW_MIRRORS],
+        "f24b384cba9935ae2e8fd3c0644312720cb6add95c49ba46f1961ae4cf0865f9",
+    ),
+    8: (
+        [urljoin(mirror, "sdo/aia/response/aia_V8_20171210_050627_response_table.txt") for mirror in _SSW_MIRRORS],
+        "0e8bc6af5a69f80ca9d4fc2a27854681b76574d59eb81d7201b7f618081f0fdd",
+    ),
+    7: (
+        [urljoin(mirror, "sdo/aia/response/aia_V7_20171129_195626_response_table.txt") for mirror in _SSW_MIRRORS],
+        "ac2171d549bd6cc6c37e13e505eef1bf0c89fc49bffd037e4ac64f0b895063ac",
+    ),
+    6: (
+        [urljoin(mirror, "sdo/aia/response/aia_V6_20141027_230030_response_table.txt") for mirror in _SSW_MIRRORS],
+        "11c148f447d4538db8fd247f74c26b4ae673355e2536f63eb48f9a267e58c7c6",
+    ),
+    4: (
+        [urljoin(mirror, "sdo/aia/response/aia_V4_20130109_204835_response_table.txt") for mirror in _SSW_MIRRORS],
+        "7e73f4effa9a8dc55f7b4993a8d181419ef555bf295c4704703ca84d7a0fc3c1",
+    ),
+    3: (
+        [urljoin(mirror, "sdo/aia/response/aia_V3_20120926_201221_response_table.txt") for mirror in _SSW_MIRRORS],
+        "0a5d2c2ed1cda18bb9fbdbd51fbf3374e042d20145150632ac95350fc99de68b",
+    ),
+    2: (
+        [urljoin(mirror, "sdo/aia/response/aia_V2_20111129_000000_response_table.txt") for mirror in _SSW_MIRRORS],
+        "d55ccd6cb3cb4bd1c688f8663f942f8a872c918a2504e5e474aa97dff45b62c9",
+    ),
+}
 
 
-def get_correction_table(*, correction_table=None):
+def get_correction_table(*, correction_table=None, calibration_version=CALIBRATION_VERSION):
     """
     Return table of degradation correction factors.
 
@@ -61,7 +100,14 @@ def get_correction_table(*, correction_table=None):
     ----------
     correction_table: `str` or `~astropy.table.QTable`, optional
         Path to correction table file or an existing correction table. If None,
-        the table will be queried from JSOC.
+        the table will be queried from JSOC. If that fails, the fixed V10
+        response table will be used.
+    calibration_version : `int`, optional
+        The version of the calibration to use when calculating the degradation.
+        By default, this is the most recent version available from JSOC. If you
+        are using a specific calibration response file, you may need to specify
+        this according to the version in that file.
+        Defaults to None which will use the most recent version available.
 
     Returns
     -------
@@ -71,11 +117,14 @@ def get_correction_table(*, correction_table=None):
     --------
     aiapy.calibrate.degradation
     """
-    if isinstance(correction_table, astropy.table.QTable):
-        return correction_table
+    # Fallback for backwards compatibility
+    if calibration_version is None:
+        calibration_version = CALIBRATION_VERSION
     if correction_table is not None:
+        if isinstance(correction_table, QTable):
+            return correction_table
         if isinstance(correction_table, str | pathlib.Path):
-            table = QTable(astropy.io.ascii.read(correction_table))
+            table = QTable(astropy_ascii.read(correction_table))
         else:
             msg = "correction_table must be a file path, an existing table, or None."
             raise ValueError(msg)
@@ -85,8 +134,18 @@ def get_correction_table(*, correction_table=None):
         # identical because the PrimeKeys for this series are WAVE_STR
         # and T_START. Without the !1=1! the query only returns the
         # latest record for each unique combination of those keywords.
-        table = drms.Client().query("aia.response[][!1=1!]", key="**ALL**")
-        table = QTable.from_pandas(table)
+        try:
+            table = drms.Client().query("aia.response[][!1=1!]", key="**ALL**")
+            table = QTable.from_pandas(table)
+        except Exception as e:  # NOQA: BLE001
+            log.warning("Unable to retrieve response table from JSOC.")
+            log.warning(f"Error: {e}")
+            log.warning(f"Falling back to fixed V{calibration_version} response table")
+            import aiapy.calibrate.util  # NOQA: PLW0406
+
+            table = QTable(
+                astropy_ascii.read(getattr(aiapy.calibrate.util, f"fetch_response_table_v{calibration_version}")())
+            )
     selected_cols = [
         "DATE",
         "VER_NUM",
@@ -194,17 +253,23 @@ def get_pointing_table(start, end):
     --------
     aiapy.calibrate.update_pointing
     """
-    q = jsoc.JSOCClient().search(
-        attrs.Time(start, end=end),
-        attrs.jsoc.Series.aia_master_pointing3h,
-    )
-    table = QTable(q)
-    if len(table.columns) == 0:
+    try:
+        q = jsoc.JSOCClient().search(
+            a.Time(start, end=end),
+            a.jsoc.Series.aia_master_pointing3h,
+        )
+        table = QTable(q)
+    except KeyError as e:
         # If there's no pointing information available between these times,
         # JSOC will raise a cryptic KeyError
         # (see https://github.com/LM-SAL/aiapy/issues/71)
         msg = f"Could not find any pointing information between {start} and {end}"
-        raise RuntimeError(msg)
+        raise RuntimeError(msg) from e
+    except Exception as e:  # NOQA: BLE001
+        log.warning("Unable to retrieve pointing table from JSOC.")
+        log.warning(f"Error: {e}")
+        log.warning("Falling back to fixed pointing table")
+        table = QTable(astropy_ascii.read(fetch_pointing_table()))
     table["T_START"] = Time(table["T_START"], scale="utc")
     table["T_STOP"] = Time(table["T_STOP"], scale="utc")
     for c in table.colnames:
@@ -223,11 +288,7 @@ def get_pointing_table(start, end):
 
 def get_error_table(error_table=None):
     if error_table is None:
-        # This is to work around a parfive bug
-        # https://github.com/Cadair/parfive/issues/121
-        os.environ["PARFIVE_DISABLE_RANGE"] = "1"
         error_table = fetch_error_table()
-        os.environ.pop("PARFIVE_DISABLE_RANGE")
     if isinstance(error_table, str | pathlib.Path):
         table = astropy.io.ascii.read(error_table)
     elif isinstance(error_table, QTable):
@@ -250,6 +311,51 @@ def get_error_table(error_table=None):
     return table
 
 
-@manager.require("error_table", *URL_HASH[ERROR_VERSION])
+@manager.require("error_table", *URL_HASH_ERROR_TABLE[ERROR_VERSION])
 def fetch_error_table():
     return manager.get("error_table")
+
+
+@manager.require("pointing_table", *URL_HASH_POINTING_TABLE)
+def fetch_pointing_table():
+    return manager.get("pointing_table")
+
+
+@manager.require("response_table_v10", *URL_HASH_RESPONSE_TABLE[10])
+def fetch_response_table_v10():
+    return manager.get("response_table_v10")
+
+
+@manager.require("response_table_v9", *URL_HASH_RESPONSE_TABLE[9])
+def fetch_response_table_v9():
+    return manager.get("response_table_v9")
+
+
+@manager.require("response_table_v8", *URL_HASH_RESPONSE_TABLE[8])
+def fetch_response_table_v8():
+    return manager.get("response_table_v8")
+
+
+@manager.require("response_table_v7", *URL_HASH_RESPONSE_TABLE[7])
+def fetch_response_table_v7():
+    return manager.get("response_table_v7")
+
+
+@manager.require("response_table_v6", *URL_HASH_RESPONSE_TABLE[6])
+def fetch_response_table_v6():
+    return manager.get("response_table_v6")
+
+
+@manager.require("response_table_v4", *URL_HASH_RESPONSE_TABLE[4])
+def fetch_response_table_v4():
+    return manager.get("response_table_v4")
+
+
+@manager.require("response_table_v3", *URL_HASH_RESPONSE_TABLE[3])
+def fetch_response_table_v3():
+    return manager.get("response_table_v3")
+
+
+@manager.require("response_table_v2", *URL_HASH_RESPONSE_TABLE[2])
+def fetch_response_table_v2():
+    return manager.get("response_table_v2")

aiapy/util/tests/test_util.py

@@ -4,6 +4,8 @@
 
 import aiapy.util
 
+pytestmark = [pytest.mark.xfail(reason="JSOC is down")]
+
 
 @pytest.mark.remote_data
 def test_sdo_location(aia_171_map) -> None:

aiapy/util/util.py

@@ -36,12 +36,16 @@ def sdo_location(time):
     """
     t = parse_time(time)
     # Query for +/- 3 seconds around the given time
-    keys = drms.Client().query(
-        f"aia.lev1[{(t - 3*u.s).utc.isot}/6s]",
-        key="T_OBS, HAEX_OBS, HAEY_OBS, HAEZ_OBS",
-    )
+    try:
+        keys = drms.Client().query(
+            f"aia.lev1[{(t - 3*u.s).utc.isot}/6s]",
+            key="T_OBS, HAEX_OBS, HAEY_OBS, HAEZ_OBS",
+        )
+    except Exception as e:
+        msg = "Unable to query the JSOC for SDO location.\n" f"Error message: {e}"
+        raise OSError(msg) from e
     if keys is None or len(keys) == 0:
-        msg = "No DRMS records near this time"
+        msg = f"No JSOC records near this time: {t}"
         raise ValueError(msg)
     # Linear interpolation between the nearest records within the returned set
     times = Time(list(keys["T_OBS"]), scale="utc")