3398 - FRA Reports - search and upload

tdrs-backend/tdpservice/data_files/migrations/0018_alter_datafile_section.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2025-02-11 22:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_files', '0017_alter_datafile_section'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datafile',
+            name='section',
+            field=models.CharField(choices=[('Tribal Closed Case Data', 'Tribal Closed Case Data'), ('Tribal Active Case Data', 'Tribal Active Case Data'), ('Tribal Aggregate Data', 'Tribal Aggregate Data'), ('Tribal Stratum Data', 'Tribal Stratum Data'), ('SSP Aggregate Data', 'Ssp Aggregate Data'), ('SSP Closed Case Data', 'Ssp Closed Case Data'), ('SSP Active Case Data', 'Ssp Active Case Data'), ('SSP Stratum Data', 'Ssp Stratum Data'), ('Active Case Data', 'Active Case Data'), ('Closed Case Data', 'Closed Case Data'), ('Aggregate Data', 'Aggregate Data'), ('Stratum Data', 'Stratum Data'), ('Work Outcomes of TANF Exiters', 'Fra Work Outcome Tanf Exiters'), ('Secondary School Attainment', 'Fra Secondry School Attainment'), ('Supplemental Work Outcomes', 'Fra Supplement Work Outcomes')], max_length=32),
+        ),
+    ]

tdrs-backend/tdpservice/data_files/models.py

@@ -119,7 +119,7 @@ class Section(models.TextChoices):
         AGGREGATE_DATA = "Aggregate Data"
         STRATUM_DATA = "Stratum Data"
 
-        FRA_WORK_OUTCOME_TANF_EXITERS = "Work Outcomes for TANF Exiters"
+        FRA_WORK_OUTCOME_TANF_EXITERS = "Work Outcomes of TANF Exiters"
         FRA_SECONDRY_SCHOOL_ATTAINMENT = "Secondary School Attainment"
         FRA_SUPPLEMENT_WORK_OUTCOMES = "Supplemental Work Outcomes"
 

tdrs-backend/tdpservice/data_files/serializers.py

@@ -118,5 +118,7 @@ def validate_file(self, file):
     def validate_section(self, section):
         """Validate the section field."""
         if DataFile.Section.is_fra(section):
-            raise serializers.ValidationError("Section cannot be FRA")
+            user = self.context.get('user')
+            if not user.has_fra_access:
+                raise serializers.ValidationError("Section cannot be FRA")
         return section

tdrs-backend/tdpservice/data_files/test/test_api.py

@@ -228,12 +228,20 @@ def test_create_data_file_file_entry(self, api_client, data_file_data, user):
         self.assert_data_file_created(response)
         self.assert_data_file_exists(data_file_data, 1, user)
 
-    def test_create_data_file_fra(self, api_client, data_file_data, user):
+    def test_create_data_file_fra_no_feat_flag(self, api_client, data_file_data, user):
         """Test ability to create data file metadata registry."""
         response = self.post_data_file_fra(api_client, data_file_data)
         assert response.data == {'section': [ErrorDetail(string='Section cannot be FRA', code='invalid')]}
         self.assert_data_file_error(response)
 
+    def test_create_data_file_fra_with_feat_flag(self, api_client, data_file_data, user):
+        """Test ability to create data file metadata registry."""
+        user.feature_flags = {"fra_reports": True}
+        user.save()
+        response = self.post_data_file_fra(api_client, data_file_data)
+        self.assert_data_file_created(response)
+        self.assert_data_file_exists(data_file_data, 1, user)
+
     def test_data_file_file_version_increment(
         self,
         api_client,

tdrs-backend/tdpservice/data_files/test/test_models.py

@@ -91,7 +91,7 @@ def test_data_files_filename_is_expected(user):
                           ('SSP Closed Case Data', 'SSP'),
                           ('Active Case Data', 'TAN'),
                           ('Aggregate Data', 'TAN'),
-                          ('Work Outcomes for TANF Exiters', 'FRA'),
+                          ('Work Outcomes of TANF Exiters', 'FRA'),
                           ('Secondary School Attainment', 'FRA'),
                           ('Supplemental Work Outcomes', 'FRA')
                           ])

tdrs-backend/tdpservice/users/migrations/0044_alter_user_feature_flags.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2025-02-11 13:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0042_user_feature_flags'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='feature_flags',
+            field=models.JSONField(blank=True, default=dict, help_text='Feature flags for this user. This is a JSON field that can be used to store key-value pairs. E.g: {"fra_reports": true}'),
+        ),
+    ]

tdrs-backend/tdpservice/users/models.py

@@ -118,14 +118,14 @@ class Meta:
     feature_flags = models.JSONField(
         default=dict,
         help_text='Feature flags for this user. This is a JSON field that can be used to store key-value pairs. ' +
-        'E.g: {"fra_access": true}',
+        'E.g: {"fra_reports": true}',
         blank=True,
     )
 
     @property
     def has_fra_access(self):
         """Return whether or not the user has FRA access."""
-        return self.feature_flags.get('fra_access', False)
+        return self.feature_flags.get('fra_reports', False)
 
     def __str__(self):
         """Return the username as the string representation of the object."""

tdrs-backend/tdpservice/users/serializers.py

@@ -115,7 +115,8 @@ class Meta:
             'date_joined',
             'access_request',
             'access_requested_date',
-            'account_approval_status'
+            'account_approval_status',
+            'feature_flags',
         ]
         read_only_fields = (
             'id',
@@ -131,6 +132,7 @@ class Meta:
             'access_request',
             'access_requested_date',
             'account_approval_status',
+            'feature_flags',
         )
 
         """Enforce first and last name to be in API call and not empty"""

tdrs-backend/tdpservice/users/test/test_models.py

@@ -79,7 +79,7 @@ def test_user_with_fra_access(client, admin_user, stt):
     """Test that a user with FRA access can only have an STT."""
     admin_user.stt = stt
     admin_user.is_superuser = True
-    admin_user.feature_flags = {"fra_access": False}
+    admin_user.feature_flags = {"fra_reports": False}
 
     admin_user.clean()
     admin_user.save()
@@ -94,7 +94,7 @@ def test_user_with_fra_access(client, admin_user, stt):
     response = client.get(f"/admin/data_files/datafile/{datafile.id}/change/")
     assert response.status_code == 302
 
-    admin_user.feature_flags = {"fra_access": True}
+    admin_user.feature_flags = {"fra_reports": True}
     admin_user.save()
 
     response = client.get(f"/admin/data_files/datafile/{datafile.id}/change/")

tdrs-frontend/src/actions/fraReports.js

@@ -0,0 +1,112 @@
+import { v4 as uuidv4 } from 'uuid'
+import axios from 'axios'
+import axiosInstance from '../axios-instance'
+import { objectToUrlParams } from '../utils/stringUtils'
+
+const BACKEND_URL = process.env.REACT_APP_BACKEND_URL
+
+export const SET_IS_LOADING_SUBMISSION_HISTORY =
+  'SET_IS_LOADING_SUBMISSION_HISTORY'
+export const SET_FRA_SUBMISSION_HISTORY = 'SET_FRA_SUBMISSION_HISTORY'
+export const SET_IS_UPLOADING_FRA_REPORT = 'SET_IS_UPLOADING_FRA_REPORT'
+
+export const getFraSubmissionHistory =
+  (
+    { stt, reportType, fiscalQuarter, fiscalYear },
+    onSuccess = () => null,
+    onError = () => null
+  ) =>
+  async (dispatch) => {
+    dispatch({
+      type: SET_IS_LOADING_SUBMISSION_HISTORY,
+      payload: { isLoadingSubmissionHistory: true },
+    })
+
+    try {
+      const requestParams = {
+        stt: stt.id,
+        file_type: reportType,
+        year: fiscalYear,
+        quarter: fiscalQuarter,
+      }
+
+      const response = await axios.get(
+        `${BACKEND_URL}/data_files/?${objectToUrlParams(requestParams)}`,
+        {
+          responseType: 'json',
+        }
+      )
+
+      dispatch({
+        type: SET_FRA_SUBMISSION_HISTORY,
+        payload: { submissionHistory: response?.data },
+      })
+
+      onSuccess()
+    } catch (error) {
+      onError(error)
+    }
+
+    dispatch({
+      type: SET_IS_LOADING_SUBMISSION_HISTORY,
+      payload: { isLoadingSubmissionHistory: false },
+    })
+  }
+
+export const uploadFraReport =
+  (
+    { stt, reportType, fiscalQuarter, fiscalYear, file, user },
+    onSuccess = () => null,
+    onError = () => null
+  ) =>
+  async (dispatch) => {
+    dispatch({
+      type: SET_IS_UPLOADING_FRA_REPORT,
+      payload: { isUploadingFraReport: true },
+    })
+
+    const formData = new FormData()
+    const fraReportData = {
+      file: file,
+      original_filename: file.name,
+      slug: uuidv4(),
+      user: user.id,
+      section: reportType,
+      year: fiscalYear,
+      stt: stt.id,
+      quarter: fiscalQuarter,
+      ssp: false,
+    }
+    for (const [key, value] of Object.entries(fraReportData)) {
+      formData.append(key, value)
+    }
+
+    try {
+      const response = await axiosInstance.post(
+        `${process.env.REACT_APP_BACKEND_URL}/data_files/`,
+        formData,
+        {
+          headers: { 'Content-Type': 'multipart/form-data' },
+          withCredentials: true,
+        }
+      )
+
+      // dispatch(
+      //   getFraSubmissionHistory({
+      //     stt,
+      //     reportType,
+      //     fiscalQuarter,
+      //     fiscalYear,
+      //   })
+      // )
+      // or, dispatch the state update if response from upload can contain updated submission history
+      onSuccess()
+    } catch (error) {
+      onError(error)
+    }
+
+    dispatch({
+      type: SET_IS_UPLOADING_FRA_REPORT,
+      payload: { isUploadingFraReport: false },
+    })
+  }

tdrs-frontend/src/components/FileUpload/FileUpload.jsx

@@ -2,7 +2,6 @@ import React, { useRef, useEffect } from 'react'
 import PropTypes from 'prop-types'
 import { useDispatch, useSelector } from 'react-redux'
 import fileTypeChecker from 'file-type-checker'
-import languageEncoding from 'detect-file-encoding-and-language'
 
 import {
   clearError,
@@ -14,7 +13,11 @@ import {
 } from '../../actions/reports'
 import Button from '../Button'
 import createFileInputErrorState from '../../utils/createFileInputErrorState'
-import { handlePreview, getTargetClassName } from './utils'
+import {
+  handlePreview,
+  getTargetClassName,
+  tryGetUTF8EncodedFile,
+} from './utils'
 
 const INVALID_FILE_ERROR =
   'We can’t process that file format. Please provide a plain text file.'
@@ -35,41 +38,6 @@ const INVALID_EXT_ERROR = (
   </>
 )
 
-// The package author suggests using a minimum of 500 words to determine the encoding. However, datafiles don't have
-// "words" so we're using bytes instead to determine the encoding. See: https://www.npmjs.com/package/detect-file-encoding-and-language
-const MIN_BYTES = 500
-
-/* istanbul ignore next */
-const tryGetUTF8EncodedFile = async function (fileBytes, file) {
-  // Create a small view of the file to determine the encoding.
-  const btyesView = new Uint8Array(fileBytes.slice(0, MIN_BYTES))
-  const blobView = new Blob([btyesView], { type: 'text/plain' })
-  try {
-    const fileInfo = await languageEncoding(blobView)
-    const bom = btyesView.slice(0, 3)
-    const hasBom = bom[0] === 0xef && bom[1] === 0xbb && bom[2] === 0xbf
-    if ((fileInfo && fileInfo.encoding !== 'UTF-8') || hasBom) {
-      const utf8Encoder = new TextEncoder()
-      const decoder = new TextDecoder(fileInfo.encoding)
-      const decodedString = decoder.decode(
-        hasBom ? fileBytes.slice(3) : fileBytes
-      )
-      const utf8Bytes = utf8Encoder.encode(decodedString)
-      return new File([utf8Bytes], file.name, file.options)
-    }
-    return file
-  } catch (error) {
-    // This is a last ditch fallback to ensure consistent functionality and also allows the unit tests to work in the
-    // same way they did before this change. When the unit tests (i.e. Node environment) call `languageEncoding` it
-    // expects a Buffer/string/URL object. When the browser calls `languageEncoding`, it expects a Blob/File object.
-    // There is not a convenient way or universal object to handle both cases. Thus, when the tests run the call to
-    // `languageEncoding`, it raises an exception and we return the file as is which is then dispatched as it would
-    // have been before this change.
-    console.error('Caught error while handling file encoding. Error:', error)
-    return file
-  }
-}
-
 const load = (file, section, input, dropTarget, dispatch) => {
   const filereader = new FileReader()
   const types = ['png', 'gif', 'jpeg']

tdrs-frontend/src/components/FileUpload/utils.jsx

@@ -1,6 +1,7 @@
 //This file contains modified versions of code from:
 //https://github.com/uswds/uswds/blob/develop/src/js/components/file-input.js
 import escapeHtml from '../../utils/escapeHtml'
+import languageEncoding from 'detect-file-encoding-and-language'
 
 export const PREFIX = 'usa'
 
@@ -95,3 +96,38 @@ export const handlePreview = (fileName, targetClassName) => {
   }
   return true
 }
+
+// The package author suggests using a minimum of 500 words to determine the encoding. However, datafiles don't have
+// "words" so we're using bytes instead to determine the encoding. See: https://www.npmjs.com/package/detect-file-encoding-and-language
+const MIN_BYTES = 500
+
+/* istanbul ignore next */
+export const tryGetUTF8EncodedFile = async function (fileBytes, file) {
+  // Create a small view of the file to determine the encoding.
+  const btyesView = new Uint8Array(fileBytes.slice(0, MIN_BYTES))
+  const blobView = new Blob([btyesView], { type: 'text/plain' })
+  try {
+    const fileInfo = await languageEncoding(blobView)
+    const bom = btyesView.slice(0, 3)
+    const hasBom = bom[0] === 0xef && bom[1] === 0xbb && bom[2] === 0xbf
+    if ((fileInfo && fileInfo.encoding !== 'UTF-8') || hasBom) {
+      const utf8Encoder = new TextEncoder()
+      const decoder = new TextDecoder(fileInfo.encoding)
+      const decodedString = decoder.decode(
+        hasBom ? fileBytes.slice(3) : fileBytes
+      )
+      const utf8Bytes = utf8Encoder.encode(decodedString)
+      return new File([utf8Bytes], file.name, file.options)
+    }
+    return file
+  } catch (error) {
+    // This is a last ditch fallback to ensure consistent functionality and also allows the unit tests to work in the
+    // same way they did before this change. When the unit tests (i.e. Node environment) call `languageEncoding` it
+    // expects a Buffer/string/URL object. When the browser calls `languageEncoding`, it expects a Blob/File object.
+    // There is not a convenient way or universal object to handle both cases. Thus, when the tests run the call to
+    // `languageEncoding`, it raises an exception and we return the file as is which is then dispatched as it would
+    // have been before this change.
+    console.error('Caught error while handling file encoding. Error:', error)
+    return file
+  }
+}