Upgrade to Django 4.2

Makefile

@@ -1,5 +1,5 @@
 prefix := /var/www/workflow
-PYTHON_VERSION:=3.10
+PYTHON_VERSION:=3.12
 
 # these are defined here because I couldn't figure out how to evaluate the commands
 # during target execution rather then when make parses the file
@@ -47,7 +47,7 @@ check: ## Check python dependencies
 	echo CONDA_PREFIX=${CONDA_PREFIX}
 
 	@python -c "import django" || echo "\nERROR: Django is not installed: www.djangoproject.com\n"
-	@python -c "import psycopg2" || echo "\nWARNING: psycopg2 is not installed: http://initd.org/psycopg\n"
+	@python -c "import psycopg" || echo "\nWARNING: psycopg is not installed: http://initd.org/psycopg\n"
 	@python -c "import stomp" || echo "\nERROR: stomp.py is not installed: http://code.google.com/p/stomppy\n"
 
 wheel/dasmon: ## create or update python wheel for service "dasmon". Clean up build/ first

README.rst

@@ -20,7 +20,7 @@ Dependencies:
  * `stomp <https://github.com/jasonrbriggs/stomp.py>`_
  * `django <https://www.djangoproject.com/>`_
  * `MySQLdb <https://sourceforge.net/projects/mysql-python/>`_ if using MySQL
- * `psycopg2 <https://www.psycopg.org/>`_ if using PostgreSQL
+ * `psycopg <https://www.psycopg.org/>`_ if using PostgreSQL
 
 It consists of 3 applications (Workflow Manager, Web Monitor, and DASMON Listener) which are deployed via docker compose.
 

conda_development.yml

@@ -4,15 +4,12 @@ dependencies:
   - flake8
   - python-dotenv
   - python-build
-  - pytest<7
-  - pytest-django<4.6
+  - pytest
+  - pytest-django
   - pytest-cov
   - pre-commit
   - sphinx
   - sphinx_rtd_theme=1.2.*
   - lxml
   - wheel
-  - pip
-  - pip:
-    - check-wheel-contents
-    - pytest-pythonpath
+  - check-wheel-contents

conda_environment.yml

@@ -3,14 +3,14 @@ channels:
   - conda-forge
   - oncat
 dependencies:
-  - python=3.10
-  - django=3.2
+  - python=3.12
+  - django=4.2
   - gunicorn
   - httplib2
   - make
   - oauthlib
-  - postgresql=14
-  - psycopg2-binary=2.9.3
+  - postgresql
+  - psycopg>=3.1.8
   - psutil
   - python-ldap>=3.1
   - requests-oauthlib
@@ -21,7 +21,7 @@ dependencies:
   - pyoncat
   - sphinx_rtd_theme=1.2.*  # readthedocs use this env file, and we need to install this theme here
   - sphinxcontrib-mermaid
+  - django-health-check
   - pip
   - pip:
-    - django-auth-ldap==4.1.0
-    - django-health-check
+    - django-auth-ldap==5.1.0

setup.cfg

@@ -4,7 +4,7 @@ ignore = E722, W504, W503
 max-line-length = 120
 
 [tool:pytest]
-python_paths = src/ src/dasmon_app src/webmon_app src/workflow_app
+pythonpath = src/ src/dasmon_app src/webmon_app src/workflow_app
 
 [coverage:run]
 source = src

src/dasmon_app/dasmon_listener/settings.py

@@ -5,7 +5,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": os.environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": os.environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": os.environ.get("DATABASE_PASS"),  # Not used with sqlite3.

src/webmon_app/docker-entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-MANAGE_PY_WEBMON="/opt/conda/lib/python3.10/site-packages/reporting/manage.py"
+MANAGE_PY_WEBMON="/opt/conda/lib/python3.12/site-packages/reporting/manage.py"
 
 # wait for postgress to be available
 until PGPASSWORD=${DATABASE_PASS} psql -h "${DATABASE_HOST}" -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -c '\q'; do

src/webmon_app/reporting/dasmon/urls.py

@@ -2,7 +2,7 @@
 """
     Define url structure
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from . import views
 
 app_name = "dasmon"

src/webmon_app/reporting/pvmon/urls.py

@@ -2,7 +2,7 @@
 """
     Define url structure
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from . import views
 
 app_name = "pvmon"

src/webmon_app/reporting/reduction/urls.py

@@ -2,7 +2,7 @@
 """
     Define url structure
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from . import views
 
 app_name = "reduction"

src/webmon_app/reporting/report/urls.py

@@ -2,7 +2,7 @@
 """
     Define url structure
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from . import views
 
 app_name = "report"

src/webmon_app/reporting/reporting_app/settings/base.py

@@ -61,10 +61,6 @@ def validate_ldap_settings(server_uri, user_dn_template):
 # to load the internationalization machinery.
 USE_I18N = True
 
-# If you set this to False, Django will not format dates, numbers and
-# calendars according to the current locale.
-USE_L10N = True
-
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = True
 
@@ -151,7 +147,7 @@ def validate_ldap_settings(server_uri, user_dn_template):
 # The DB settings are defined the same as in the workflow manager
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": environ.get("DATABASE_PASS"),  # Not used with sqlite3.

src/webmon_app/reporting/reporting_app/urls.py

@@ -2,7 +2,6 @@
 """
     Define url structure
 """
-# from django.conf.urls import include, re_path
 from django.urls import include, path
 from django.contrib.auth.decorators import login_required
 

src/webmon_app/reporting/users/urls.py

@@ -2,7 +2,7 @@
 """
     Define url structure
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from . import views
 
 app_name = "users"

src/workflow_app/workflow/database/settings.py

@@ -3,7 +3,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": os.environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": os.environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": os.environ.get("DATABASE_PASS"),  # Not used with sqlite3.

tests/conftest.py

@@ -1,6 +1,6 @@
 # third-party imports
 from dotenv import dotenv_values
-import psycopg2
+import psycopg
 import pytest
 import stomp
 
@@ -67,8 +67,8 @@ def db_connection():
     """Database connection with config from env files"""
     config = {**dotenv_values(".env"), **dotenv_values(".env.ci")}
     assert config
-    conn = psycopg2.connect(
-        database=config["DATABASE_NAME"],
+    conn = psycopg.connect(
+        dbname=config["DATABASE_NAME"],
         user=config["DATABASE_USER"],
         password=config["DATABASE_PASS"],
         port=config["DATABASE_PORT"],

tests/test_PostProcessWorkflow.py

@@ -1,4 +1,4 @@
-import psycopg2
+import psycopg
 import requests
 import time
 from django.conf import settings
@@ -13,8 +13,8 @@ class TestPostProcessingWorkflow:
     def setup_class(cls):
         config = {**dotenv_values(".env"), **dotenv_values(".env.ci")}
         assert config
-        cls.conn = psycopg2.connect(
-            database=config["DATABASE_NAME"],
+        cls.conn = psycopg.connect(
+            dbname=config["DATABASE_NAME"],
             user=config["DATABASE_USER"],
             password=config["DATABASE_PASS"],
             port=config["DATABASE_PORT"],

tests/test_ReductionSetupPageView.py

@@ -1,6 +1,6 @@
 import os
 import time
-import psycopg2
+import psycopg
 import pytest
 import requests
 import subprocess
@@ -49,16 +49,18 @@ def getReductionScriptContents(self):
         ).decode()
 
     def initReductionGroup(self, conn, cursor):
-        cursor.execute("SELECT * from reduction_reductionproperty WHERE instrument_id = 3;")
+        cursor.execute("SELECT id FROM report_instrument where name = %s;", ("arcs",))
+        inst_id = cursor.fetchone()[0]
+        cursor.execute("SELECT * from reduction_reductionproperty WHERE instrument_id = %s;", (inst_id,))
         if cursor.fetchone() is None:
             timestamp = datetime.datetime.now()
             cursor.execute(
                 "INSERT INTO reduction_reductionproperty (instrument_id, key, value, timestamp) VALUES(%s, %s, %s, %s)",
-                (3, "grouping", "/SNS/ARCS/shared/autoreduce/ARCS_2X1_grouping.xml", timestamp),
+                (inst_id, "grouping", "/SNS/ARCS/shared/autoreduce/ARCS_2X1_grouping.xml", timestamp),
             )
             conn.commit()
 
-        cursor.execute("SELECT * from reduction_choice WHERE instrument_id = 3;")
+        cursor.execute("SELECT * from reduction_choice WHERE instrument_id = %s;", (inst_id,))
         if cursor.fetchone() is None:
             cursor.execute("SELECT * FROM reduction_reductionproperty WHERE key = 'grouping';")
             props = cursor.fetchone()
@@ -108,8 +110,8 @@ def testReduction(self, instrument_scientist_client):
 
         assert "this is a template" not in self.getReductionScriptContents()
 
-        conn = psycopg2.connect(
-            database="workflow",
+        conn = psycopg.connect(
+            dbname="workflow",
             user="workflow",
             password="workflow",
             port="5432",

tests/test_SMS_messages.py

@@ -36,16 +36,18 @@ def clear_run(self, conn, run_number):
             return
 
         cursor.execute(
-            "SELECT id FROM report_datarun WHERE instrument_id_id = %s AND run_number = %s;", (inst_id, run_number)
+            "SELECT id FROM report_datarun WHERE instrument_id_id = %s AND run_number = %s;", (inst_id[0], run_number)
         )
         run_id = cursor.fetchone()
         if run_id is None:
             return
 
+        run_id = run_id[0]
+
         db_utils.clear_previous_runstatus(conn, run_id)
-        cursor.execute("DELETE FROM report_workflowsummary WHERE run_id_id = %s;", run_id)
-        cursor.execute("DELETE FROM report_instrumentstatus WHERE last_run_id_id = %s;", run_id)
-        cursor.execute("DELETE FROM report_datarun WHERE id = %s;", (run_id))
+        cursor.execute("DELETE FROM report_workflowsummary WHERE run_id_id = %s;", (run_id,))
+        cursor.execute("DELETE FROM report_instrumentstatus WHERE last_run_id_id = %s;", (run_id,))
+        cursor.execute("DELETE FROM report_datarun WHERE id = %s;", (run_id,))
         conn.commit()
         cursor.close()
 

tests/test_SubmitPostprocessing.py

@@ -1,6 +1,6 @@
 import requests
 import time
-import psycopg2
+import psycopg
 import pytest
 
 INSTRUMENT = "arcs"
@@ -12,8 +12,8 @@
 class TestPostProcessingAdminView:
     def setup_class(cls):
         # connect to DB
-        cls.conn = psycopg2.connect(
-            database="workflow",
+        cls.conn = psycopg.connect(
+            dbname="workflow",
             user="workflow",
             password="workflow",
             port="5432",

tests/test_livedata.py

@@ -2,7 +2,7 @@
 import os
 import hashlib
 
-import psycopg2
+import psycopg
 import requests
 
 LIVEDATA_TEST_URL = "https://172.16.238.222"
@@ -17,8 +17,8 @@ class TestLiveDataServer:
     @classmethod
     def setup_class(cls):
         """Clean the database before running tests"""
-        conn = psycopg2.connect(
-            database=os.environ.get("DATABASE_NAME", "workflow"),
+        conn = psycopg.connect(
+            dbname=os.environ.get("DATABASE_NAME", "workflow"),
             user=os.environ.get("DATABASE_USER", "workflow"),
             password=os.environ.get("DATABASE_PASS", "workflow"),
             port=os.environ.get("DATABASE_PORT", 5432),