3398 - FRA Reports - search and upload

tdrs-backend/tdpservice/conftest.py

@@ -316,6 +316,23 @@ def data_file_data(base_data_file_data, data_file):
     }
 
 
+@pytest.fixture
+def csv_data_file(data_analyst, fake_file):
+    """Return a CSV data file for testing FRA uploads."""
+    return {
+        'file': create_temporary_file(fake_file, 'report.csv'),
+        "original_filename": 'report.csv',
+        "slug": str(uuid.uuid4()),
+        "extension": "txt",
+        "section": "Active Case Data",
+        "user": str(data_analyst.id),
+        "quarter": "Q1",
+        "year": 2020,
+        "stt": int(data_analyst.stt.id),
+        "ssp": False,
+    }
+
+
 @pytest.fixture
 def regional_data_file_data(base_regional_data_file_data, data_file):
     """Return data file creation data for a reigon."""

tdrs-backend/tdpservice/data_files/migrations/0018_alter_datafile_section.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2025-02-11 22:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_files', '0017_alter_datafile_section'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datafile',
+            name='section',
+            field=models.CharField(choices=[('Tribal Closed Case Data', 'Tribal Closed Case Data'), ('Tribal Active Case Data', 'Tribal Active Case Data'), ('Tribal Aggregate Data', 'Tribal Aggregate Data'), ('Tribal Stratum Data', 'Tribal Stratum Data'), ('SSP Aggregate Data', 'Ssp Aggregate Data'), ('SSP Closed Case Data', 'Ssp Closed Case Data'), ('SSP Active Case Data', 'Ssp Active Case Data'), ('SSP Stratum Data', 'Ssp Stratum Data'), ('Active Case Data', 'Active Case Data'), ('Closed Case Data', 'Closed Case Data'), ('Aggregate Data', 'Aggregate Data'), ('Stratum Data', 'Stratum Data'), ('Work Outcomes of TANF Exiters', 'Fra Work Outcome Tanf Exiters'), ('Secondary School Attainment', 'Fra Secondry School Attainment'), ('Supplemental Work Outcomes', 'Fra Supplement Work Outcomes')], max_length=32),
+        ),
+    ]

tdrs-backend/tdpservice/data_files/models.py

@@ -119,7 +119,7 @@ class Section(models.TextChoices):
         AGGREGATE_DATA = "Aggregate Data"
         STRATUM_DATA = "Stratum Data"
 
-        FRA_WORK_OUTCOME_TANF_EXITERS = "Work Outcomes for TANF Exiters"
+        FRA_WORK_OUTCOME_TANF_EXITERS = "Work Outcomes of TANF Exiters"
         FRA_SECONDRY_SCHOOL_ATTAINMENT = "Secondary School Attainment"
         FRA_SUPPLEMENT_WORK_OUTCOMES = "Supplemental Work Outcomes"
 

tdrs-backend/tdpservice/data_files/serializers.py

@@ -108,15 +108,22 @@ def update(self, instance, validated_data):
         """Throw an error if a user tries to update a data_file."""
         raise ImmutabilityError(instance, validated_data)
 
-    def validate_file(self, file):
+    def validate(self, data):
         """Perform all validation steps on a given file."""
         user = self.context.get('user')
-        validate_file_extension(file.name)
-        validate_file_infection(file, file.name, user)
-        return file
+        file = data['file'] if 'file' in data else None
+        section = data['section'] if 'section' in data else None
+
+        if file and section:
+            validate_file_extension(file.name, is_fra=DataFile.Section.is_fra(section))
+            validate_file_infection(file, file.name, user)
+
+        return data
 
     def validate_section(self, section):
         """Validate the section field."""
         if DataFile.Section.is_fra(section):
-            raise serializers.ValidationError("Section cannot be FRA")
+            user = self.context.get('user')
+            if not user.has_fra_access:
+                raise serializers.ValidationError("Section cannot be FRA")
         return section

tdrs-backend/tdpservice/data_files/test/test_api.py

@@ -228,12 +228,20 @@ def test_create_data_file_file_entry(self, api_client, data_file_data, user):
         self.assert_data_file_created(response)
         self.assert_data_file_exists(data_file_data, 1, user)
 
-    def test_create_data_file_fra(self, api_client, data_file_data, user):
+    def test_create_data_file_fra_no_feat_flag(self, api_client, data_file_data, user):
         """Test ability to create data file metadata registry."""
         response = self.post_data_file_fra(api_client, data_file_data)
         assert response.data == {'section': [ErrorDetail(string='Section cannot be FRA', code='invalid')]}
         self.assert_data_file_error(response)
 
+    def test_create_data_file_fra_with_feat_flag(self, api_client, csv_data_file, user):
+        """Test ability to create data file metadata registry."""
+        user.feature_flags = {"fra_reports": True}
+        user.save()
+        response = self.post_data_file_fra(api_client, csv_data_file)
+        self.assert_data_file_created(response)
+        self.assert_data_file_exists(csv_data_file, 1, user)
+
     def test_data_file_file_version_increment(
         self,
         api_client,

tdrs-backend/tdpservice/data_files/test/test_models.py

@@ -91,7 +91,7 @@ def test_data_files_filename_is_expected(user):
                           ('SSP Closed Case Data', 'SSP'),
                           ('Active Case Data', 'TAN'),
                           ('Aggregate Data', 'TAN'),
-                          ('Work Outcomes for TANF Exiters', 'FRA'),
+                          ('Work Outcomes of TANF Exiters', 'FRA'),
                           ('Secondary School Attainment', 'FRA'),
                           ('Supplemental Work Outcomes', 'FRA')
                           ])

tdrs-backend/tdpservice/data_files/test/test_serializers.py

@@ -53,7 +53,7 @@ def test_immutability_of_data_file(data_file_instance):
     with pytest.raises(ImmutabilityError):
         serializer = DataFileSerializer(
             data_file_instance, data={
-                "original_filename": "BadGuy.js"
+                "original_filename": "BadGuy.js",
             },
             partial=True
         )

tdrs-backend/tdpservice/data_files/validators.py

@@ -12,12 +12,17 @@
 # The default set of Extensions allowed for an uploaded file
 # Supports regex patterns as defined by standard library
 # https://docs.python.org/3/library/re.html#regular-expression-syntax
-ALLOWED_FILE_EXTENSIONS = [
+ALLOWED_TANF_FILE_EXTENSIONS = [
     r'^(ms\d{2})$',  # Files ending in .MS## where # is a digit 0-9
     r'^(ts\d{2,3})$',  # Files ending in .TS## or .TS### where # is a digit 0-9
     'txt',  # plain text files
 ]
 
+ALLOWED_FRA_FILE_EXTENSIONS = [
+    'csv',
+    'xlsx'
+]
+
 
 def _get_unsupported_msg(_type, value, supported_options):
     """Construct a message to convey an unsupported operation."""
@@ -27,21 +32,23 @@ def _get_unsupported_msg(_type, value, supported_options):
     )
 
 
-def validate_file_extension(file_name: str):
+def validate_file_extension(file_name: str, is_fra: bool = False):
     """Validate the file extension of a file is in our supported list."""
     file_extension = (
         file_name.split('.')[-1].lower() if '.' in file_name else None
     )
 
-    allowed_ext_patterns = '|'.join(ALLOWED_FILE_EXTENSIONS)
+    allowed_list = ALLOWED_FRA_FILE_EXTENSIONS if is_fra else ALLOWED_TANF_FILE_EXTENSIONS
+
+    allowed_ext_patterns = '|'.join(allowed_list)
     if (
         file_extension is not None
         and not re.match(allowed_ext_patterns, file_extension)
     ):
         msg = _get_unsupported_msg(
             'file extension',
             file_extension,
-            ALLOWED_FILE_EXTENSIONS
+            allowed_list
         )
         raise ValidationError(msg)
 

tdrs-backend/tdpservice/users/migrations/0044_alter_user_feature_flags.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2025-02-11 13:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0043_create_regions_m2m_del_region'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='feature_flags',
+            field=models.JSONField(blank=True, default=dict, help_text='Feature flags for this user. This is a JSON field that can be used to store key-value pairs. E.g: {"fra_reports": true}'),
+        ),
+    ]

tdrs-backend/tdpservice/users/models.py

@@ -129,14 +129,14 @@ class Meta:
     feature_flags = models.JSONField(
         default=dict,
         help_text='Feature flags for this user. This is a JSON field that can be used to store key-value pairs. ' +
-        'E.g: {"fra_access": true}',
+        'E.g: {"fra_reports": true}',
         blank=True,
     )
 
     @property
     def has_fra_access(self):
         """Return whether or not the user has FRA access."""
-        return self.feature_flags.get('fra_access', False)
+        return self.feature_flags.get('fra_reports', False)
 
     def __str__(self):
         """Return the username as the string representation of the object."""

tdrs-backend/tdpservice/users/serializers.py

@@ -115,7 +115,8 @@ class Meta:
             'date_joined',
             'access_request',
             'access_requested_date',
-            'account_approval_status'
+            'account_approval_status',
+            'feature_flags',
         ]
         read_only_fields = (
             'id',
@@ -131,6 +132,7 @@ class Meta:
             'access_request',
             'access_requested_date',
             'account_approval_status',
+            'feature_flags',
         )
 
         """Enforce first and last name to be in API call and not empty"""

tdrs-backend/tdpservice/users/test/test_models.py

@@ -80,7 +80,7 @@ def test_user_with_fra_access(client, admin_user, stt):
     """Test that a user with FRA access can only have an STT."""
     admin_user.stt = stt
     admin_user.is_superuser = True
-    admin_user.feature_flags = {"fra_access": False}
+    admin_user.feature_flags = {"fra_reports": False}
 
     admin_user.clean()
     admin_user.save()
@@ -95,7 +95,7 @@ def test_user_with_fra_access(client, admin_user, stt):
     response = client.get(f"/admin/data_files/datafile/{datafile.id}/change/")
     assert response.status_code == 302
 
-    admin_user.feature_flags = {"fra_access": True}
+    admin_user.feature_flags = {"fra_reports": True}
     admin_user.save()
 
     response = client.get(f"/admin/data_files/datafile/{datafile.id}/change/")

tdrs-frontend/src/actions/fraReports.js

@@ -0,0 +1,112 @@
+import { v4 as uuidv4 } from 'uuid'
+import axios from 'axios'
+import axiosInstance from '../axios-instance'
+import { objectToUrlParams } from '../utils/stringUtils'
+
+const BACKEND_URL = process.env.REACT_APP_BACKEND_URL
+
+export const SET_IS_LOADING_SUBMISSION_HISTORY =
+  'SET_IS_LOADING_SUBMISSION_HISTORY'
+export const SET_FRA_SUBMISSION_HISTORY = 'SET_FRA_SUBMISSION_HISTORY'
+export const SET_IS_UPLOADING_FRA_REPORT = 'SET_IS_UPLOADING_FRA_REPORT'
+
+export const getFraSubmissionHistory =
+  (
+    { stt, reportType, fiscalQuarter, fiscalYear },
+    onSuccess = () => null,
+    onError = () => null
+  ) =>
+  async (dispatch) => {
+    dispatch({
+      type: SET_IS_LOADING_SUBMISSION_HISTORY,
+      payload: { isLoadingSubmissionHistory: true },
+    })
+
+    try {
+      const requestParams = {
+        stt: stt.id,
+        file_type: reportType,
+        year: fiscalYear,
+        quarter: fiscalQuarter,
+      }
+
+      const response = await axios.get(
+        `${BACKEND_URL}/data_files/?${objectToUrlParams(requestParams)}`,
+        {
+          responseType: 'json',
+        }
+      )
+
+      dispatch({
+        type: SET_FRA_SUBMISSION_HISTORY,
+        payload: { submissionHistory: response?.data },
+      })
+
+      onSuccess()
+    } catch (error) {
+      onError(error)
+    } finally {
+      dispatch({
+        type: SET_IS_LOADING_SUBMISSION_HISTORY,
+        payload: { isLoadingSubmissionHistory: false },
+      })
+    }
+  }
+
+export const uploadFraReport =
+  (
+    { stt, reportType, fiscalQuarter, fiscalYear, file, user },
+    onSuccess = () => null,
+    onError = () => null
+  ) =>
+  async (dispatch) => {
+    dispatch({
+      type: SET_IS_UPLOADING_FRA_REPORT,
+      payload: { isUploadingFraReport: true },
+    })
+
+    const formData = new FormData()
+    const fraReportData = {
+      file: file,
+      original_filename: file.name,
+      slug: uuidv4(),
+      user: user.id,
+      section: reportType,
+      year: fiscalYear,
+      stt: stt.id,
+      quarter: fiscalQuarter,
+      ssp: false,
+    }
+    for (const [key, value] of Object.entries(fraReportData)) {
+      formData.append(key, value)
+    }
+
+    try {
+      const response = await axiosInstance.post(
+        `${process.env.REACT_APP_BACKEND_URL}/data_files/`,
+        formData,
+        {
+          headers: { 'Content-Type': 'multipart/form-data' },
+          withCredentials: true,
+        }
+      )
+
+      // dispatch(
+      //   getFraSubmissionHistory({
+      //     stt,
+      //     reportType,
+      //     fiscalQuarter,
+      //     fiscalYear,
+      //   })
+      // )
+      // or, dispatch the state update if response from upload can contain updated submission history
+      onSuccess()
+    } catch (error) {
+      onError(error)
+    } finally {
+      dispatch({
+        type: SET_IS_UPLOADING_FRA_REPORT,
+        payload: { isUploadingFraReport: false },
+      })
+    }
+  }