respond to pr comments with new field descriptions

docs/release_notes.rst

@@ -12,7 +12,7 @@ New Data
 SEC Form 10-K Parent-Subsidiary Ownership
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* We have added some new tables describing the parent-subsidary company ownership
+* We have added some new tables describing the parent-subsidiary company ownership
   relationships reported in the
   `SEC's Form 10-K <https://en.wikipedia.org/wiki/Form_10-K>`__, Exhibit 21
   "Subsidiaries of the Registrant". Where possible these tables link the SEC filers or
@@ -35,6 +35,15 @@ SEC Form 10-K Parent-Subsidiary Ownership
   * :ref:`core_sec10k__quarterly_exhibit_21_company_ownership`
   * :ref:`core_sec10k__quarterly_company_information`
 
+FERC 1
+~~~~~~
+* Added three new output tables containing granular utility accounting data.
+  See :pr:`4057`, :issue:`3642` and the table descriptions in the data dictionary:
+
+  * :ref:`out_ferc1__yearly_detailed_income_statements`
+  * :ref:`out_ferc1__yearly_detailed_balance_sheet_assets`
+  * :ref:`out_ferc1__yearly_detailed_balance_sheet_liabilities`
+
 New Data Coverage
 ^^^^^^^^^^^^^^^^^
 

migrations/versions/ad6ef3a763ba_add_my_cool_ferc1_detailed_tables.py → migrations/versions/b2acd10c5d2b_add_my_cool_ferc1_detailed_tables.py

@@ -1,16 +1,16 @@
 """Add my cool ferc1 detailed tables
 
-Revision ID: ad6ef3a763ba
+Revision ID: b2acd10c5d2b
 Revises: 1e2ec7bf2b64
-Create Date: 2025-02-12 17:45:19.841193
+Create Date: 2025-02-12 19:01:13.986694
 
 """
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'ad6ef3a763ba'
+revision = 'b2acd10c5d2b'
 down_revision = '1e2ec7bf2b64'
 branch_labels = None
 depends_on = None
@@ -27,7 +27,7 @@ def upgrade() -> None:
     sa.Column('plant_status', sa.Text(), nullable=True, comment='Utility plant financial status (in service, future, leased, total).'),
     sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'),
     sa.Column('utility_type_other', sa.Text(), nullable=True, comment='Freeform description of type of utility reported in one of the other three other utility_type sections in the core_ferc1__yearly_utility_plant_summary_sched200 table. This field is reported only in the DBF reporting years (1994-2020).'),
-    sa.Column('in_rate_base', sa.Boolean(), nullable=True, comment="Whether or not a record from the detailed FERC1 accounting tables should be considered allowable in a utility's rate base based on utility accounting standards. This flag was mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes varies based on the utility_type plant_status or plant_function as well."),
+    sa.Column('in_rate_base', sa.Boolean(), nullable=True, comment="Whether or not a record from the detailed FERC1 accounting tables should be considered allowable in a utility's rate base based on utility accounting standards. This flag was mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes varies based on the utility_type, plant_status or plant_function."),
     sa.Column('rate_base_category', sa.Enum('other_plant', 'nuclear', 'transmission', 'net_nuclear_fuel', 'distribution', 'steam', 'experimental_plant', 'net_working_capital', 'general_plant', 'regional_transmission_and_market_operation', 'other_production', 'other_noncurrent_liabilities', 'hydro', 'net_utility_plant', 'intangible_plant', 'other_deferred_debits_and_credits', 'net_regulatory_assets', 'net_ADIT', 'asset_retirement_costs', 'utility_plant', 'electric_plant_leased_to_others', 'electric_plant_held_for_future_use', 'non_utility_plant', 'construction_work_in_progress', 'AROs', 'correction'), nullable=True, comment='A category of asset or liability that RMI compiled to use as a shorthand for various types of utility assets. These tags were compiled manually based on the xbrl_factoid and sometimes varies based on the utility_type, plant_function or plant_status as well.'),
     sa.Column('table_name', sa.Text(), nullable=True, comment='The name of the PUDL database table where a given record originated from.'),
     sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_detailed_balance_sheet_assets_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities'))
@@ -38,7 +38,7 @@ def upgrade() -> None:
     sa.Column('xbrl_factoid', sa.Text(), nullable=True, comment='The name of type of value which is a derivative of the XBRL fact name.'),
     sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'),
     sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'),
-    sa.Column('in_rate_base', sa.Boolean(), nullable=True, comment="Whether or not a record from the detailed FERC1 accounting tables should be considered allowable in a utility's rate base based on utility accounting standards. This flag was mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes varies based on the utility_type plant_status or plant_function as well."),
+    sa.Column('in_rate_base', sa.Boolean(), nullable=True, comment="Whether or not a record from the detailed FERC1 accounting tables should be considered allowable in a utility's rate base based on utility accounting standards. This flag was mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes varies based on the utility_type, plant_status or plant_function."),
     sa.Column('rate_base_category', sa.Enum('other_plant', 'nuclear', 'transmission', 'net_nuclear_fuel', 'distribution', 'steam', 'experimental_plant', 'net_working_capital', 'general_plant', 'regional_transmission_and_market_operation', 'other_production', 'other_noncurrent_liabilities', 'hydro', 'net_utility_plant', 'intangible_plant', 'other_deferred_debits_and_credits', 'net_regulatory_assets', 'net_ADIT', 'asset_retirement_costs', 'utility_plant', 'electric_plant_leased_to_others', 'electric_plant_held_for_future_use', 'non_utility_plant', 'construction_work_in_progress', 'AROs', 'correction'), nullable=True, comment='A category of asset or liability that RMI compiled to use as a shorthand for various types of utility assets. These tags were compiled manually based on the xbrl_factoid and sometimes varies based on the utility_type, plant_function or plant_status as well.'),
     sa.Column('table_name', sa.Text(), nullable=True, comment='The name of the PUDL database table where a given record originated from.'),
     sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_detailed_balance_sheet_liabilities_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities'))
@@ -49,10 +49,10 @@ def upgrade() -> None:
     sa.Column('xbrl_factoid', sa.Text(), nullable=True, comment='The name of type of value which is a derivative of the XBRL fact name.'),
     sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'),
     sa.Column('plant_function', sa.Text(), nullable=True, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'),
-    sa.Column('revenue_requirement_technology', sa.Enum('administrative', 'common_plant_electric', 'customer_accounts', 'customer_service', 'distribution', 'general', 'hydraulic_production', 'hydraulic_production_conventional', 'hydraulic_production_pumped_storage', 'intangible', 'nuclear_production', 'other', 'other_electric_plant', 'other_power_supply', 'other_production', 'purchased_power', 'regional_transmission_and_market_operation', 'sales', 'steam_production', 'transmission'), nullable=True, comment="The technology type associated with components of a utility's revenue requirement. These categories were mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes vary based on the utility_type or plant_function as well."),
+    sa.Column('revenue_requirement_technology', sa.Enum('administrative', 'common_plant_electric', 'customer_accounts', 'customer_service', 'distribution', 'general', 'hydraulic_production', 'hydraulic_production_conventional', 'hydraulic_production_pumped_storage', 'intangible', 'nuclear_production', 'other', 'other_electric_plant', 'other_power_supply', 'other_production', 'purchased_power', 'regional_transmission_and_market_operation', 'sales', 'steam_production', 'transmission'), nullable=True, comment="The technology type associated with components of a utility's revenue requirement. These categories were mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes vary based on the utility_type or plant_function as well. This column is intended to be used to aggregate this table."),
     sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'),
-    sa.Column('in_revenue_requirement_review', sa.Boolean(), nullable=True, comment="Whether or not a record from the detailed income statement data should be considered in a utility's revenue requirement. This flag was mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes varies based on the utility_type or plant_function as well."),
-    sa.Column('revenue_requirement_category', sa.Enum('depreciation_amortization_depletion', 'depreciation_arc', 'fuel', 'investment_tax_credit', 'maintenance', 'non_fuel_operation', 'other', 'purchased_power', 'regulatory_debits_credits', 'taxes'), nullable=True, comment='A category of revenue requirement components. These categories were mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes vary based on the utility_type or plant_function as well.'),
+    sa.Column('in_revenue_requirement', sa.Boolean(), nullable=True, comment="Whether or not a record from the detailed income statement data is typically included in a utility's revenue requirement. This flag was mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes varies based on the utility_type or plant_function."),
+    sa.Column('revenue_requirement_category', sa.Enum('depreciation_amortization_depletion', 'depreciation_arc', 'fuel', 'investment_tax_credit', 'maintenance', 'non_fuel_operation', 'other', 'purchased_power', 'regulatory_debits_credits', 'taxes'), nullable=True, comment="The category of revenue requirement associated with each component of utility'sincome statements. These categories were mannually compiled by RMI utility accounting experts based on the xbrl_factoid and sometimes vary based on the utility_type or plant_function. This column is intended to be used to aggregate this table."),
     sa.Column('table_name', sa.Text(), nullable=True, comment='The name of the PUDL database table where a given record originated from.'),
     sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_detailed_income_statements_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities'))
     )

src/pudl/metadata/fields.py

@@ -2222,17 +2222,17 @@
             "be considered allowable in a utility's rate base based on utility "
             "accounting standards. "
             "This flag was mannually compiled by RMI utility accounting experts "
-            "based on the xbrl_factoid and sometimes varies based on the utility_type "
-            "plant_status or plant_function as well."
+            "based on the xbrl_factoid and sometimes varies based on the utility_type, "
+            "plant_status or plant_function."
         ),
     },
-    "in_revenue_requirement_review": {
+    "in_revenue_requirement": {
         "type": "boolean",
         "description": (
-            "Whether or not a record from the detailed income statement data should be "
-            "considered in a utility's revenue requirement. This flag was mannually "
+            "Whether or not a record from the detailed income statement data is typically "
+            "included in a utility's revenue requirement. This flag was mannually "
             "compiled by RMI utility accounting experts based on the xbrl_factoid and "
-            "sometimes varies based on the utility_type or plant_function as well."
+            "sometimes varies based on the utility_type or plant_function."
         ),
     },
     "inactive_accounts_included": {
@@ -4035,10 +4035,12 @@
     "revenue_requirement_category": {
         "type": "string",
         "description": (
-            "A category of revenue requirement components. "
+            "The category of revenue requirement associated with each component of utility's"
+            "income statements. "
             "These categories were mannually compiled by RMI utility accounting experts "
             "based on the xbrl_factoid and sometimes vary based on the utility_type or "
-            "plant_function as well."
+            "plant_function. This column is intended to be used to aggregate this "
+            "table."
         ),
         "constraints": {
             "enum": [
@@ -4062,7 +4064,8 @@
             "revenue requirement. "
             "These categories were mannually compiled by RMI utility accounting experts "
             "based on the xbrl_factoid and sometimes vary based on the utility_type or "
-            "plant_function as well."
+            "plant_function as well. This column is intended to be used to aggregate this "
+            "table."
         ),
         "constraints": {
             "enum": [

src/pudl/metadata/resources/ferc1.py

@@ -1689,11 +1689,11 @@
     "out_ferc1__yearly_detailed_income_statements": {
         "description": (
             "This table contains granular accounting data from utilities' income statements."
-            "This table is derived from four FERC Form 1 accounting tables with nested calculations:\n"
-            " - `core_ferc1__yearly_income_statements_sched114`\n"
-            " - `core_ferc1__yearly_depreciation_summary_sched336`\n"
-            " - `core_ferc1__yearly_operating_expenses_sched320`\n"
-            " - `core_ferc1__yearly_operating_revenues_sched300`\n"
+            "This table is derived from four FERC Form 1 accounting tables with nested calculations:\n\n"
+            " * `core_ferc1__yearly_income_statements_sched114`\n"
+            " * `core_ferc1__yearly_depreciation_summary_sched336`\n"
+            " * `core_ferc1__yearly_operating_expenses_sched320`\n"
+            " * `core_ferc1__yearly_operating_revenues_sched300`\n\n"
             "We reconciled the nested calculations within these tables and then identified the "
             "most granular data across the tables.\n"
             "We applied slight modifications to two columns (utility_type & plant_function)"
@@ -1714,7 +1714,7 @@
                 "plant_function",
                 "revenue_requirement_technology",
                 "dollar_value",
-                "in_revenue_requirement_review",
+                "in_revenue_requirement",
                 "revenue_requirement_category",
                 "table_name",
             ]
@@ -1726,11 +1726,11 @@
     "out_ferc1__yearly_detailed_balance_sheet_assets": {
         "description": (
             "This table contains granular accounting data from utilities' balance sheet assets. "
-            "This table is derived from four FERC Form 1 accounting tables with nested calculations:\n"
-            " - `core_ferc1__yearly_balance_sheet_assets_sched110`\n"
-            " - `core_ferc1__yearly_utility_plant_summary_sched200`\n"
-            " - `core_ferc1__yearly_plant_in_service_sched204`\n"
-            " - `core_ferc1__yearly_depreciation_by_function_sched219`\n"
+            "This table is derived from four FERC Form 1 accounting tables with nested calculations:\n\n"
+            " * `core_ferc1__yearly_balance_sheet_assets_sched110`\n"
+            " * `core_ferc1__yearly_utility_plant_summary_sched200`\n"
+            " * `core_ferc1__yearly_plant_in_service_sched204`\n"
+            " * `core_ferc1__yearly_depreciation_by_function_sched219`\n\n"
             "We reconciled the nested calculations within these tables and then identified the "
             "most granular data across the tables.\n"
             "We applied slight modifications to three columns (utility_type, plant_function & plant_status) "
@@ -1764,9 +1764,9 @@
     "out_ferc1__yearly_detailed_balance_sheet_liabilities": {
         "description": (
             "This table contains granular accounting data from utilities' balance sheet liabilities. "
-            "This table is derived from two FERC Form 1 accounting tables with nested calculations:\n"
-            " - `core_ferc1__yearly_balance_sheet_liabilities_sched110`\n"
-            " - `core_ferc1__yearly_retained_earnings_sched118`\n"
+            "This table is derived from two FERC Form 1 accounting tables with nested calculations:\n\n"
+            " * `core_ferc1__yearly_balance_sheet_liabilities_sched110`\n"
+            " * `core_ferc1__yearly_retained_earnings_sched118`\n\n"
             "We reconciled the nested calculations within these tables and then identified the "
             "most granular data across the tables.\n"
             "We applied slight modifications to three columns (utility_type, plant_function & plant_status) "

src/pudl/output/ferc1.py

@@ -2669,7 +2669,7 @@ def _check(df):
     return _check
 
 
-_checks = [
+_tag_checks = [
     make_check_tag_propagation(spec) for spec in check_specs_detailed_tables_tags
 ] + [make_check_correction_tags(spec) for spec in check_specs_detailed_tables_tags]
 
@@ -2801,7 +2801,12 @@ def make_idx_check(spec: Ferc1DetailedCheckSpec) -> AssetChecksDefinition:
     """Turn the Ferc1DetailedCheckSpec into an actual Dagster asset check."""
 
     @asset_check(asset=spec.asset, blocking=True)
-    def _idk_check(context, df):
+    def _idx_check(df):
+        """Check the primary keys of this table.
+
+        We do this as an asset check instead of actually setting them as primary keys
+        in the db schema because there are many expected nulls in these columns.
+        """
         idx = spec.idx
         dupes = df[df.duplicated(idx, keep=False)]
         if "plant_function" in dupes:
@@ -2817,10 +2822,10 @@ def _idk_check(context, df):
             )
         return AssetCheckResult(passed=True)
 
-    return _idk_check
+    return _idx_check
 
 
-_checks = [make_idx_check(spec) for spec in check_specs]
+_idx_checks = [make_idx_check(spec) for spec in check_specs]
 
 
 def prep_cash_working_capital(

src/pudl/package_data/ferc1/xbrl_factoid_revenue_requirement_tags.csv

@@ -1,4 +1,4 @@
-table_name,xbrl_factoid,in_revenue_requirement_review,revenue_requirement_category,revenue_requirement_technology,plant_function,utility_type,plant_status
+table_name,xbrl_factoid,in_revenue_requirement,revenue_requirement_category,revenue_requirement_technology,plant_function,utility_type,plant_status
 core_ferc1__yearly_operating_expenses_sched320,operation_supervision_and_engineering_steam_power_generation,True,non_fuel_operation,steam_production,,,
 core_ferc1__yearly_operating_expenses_sched320,fuel_steam_power_generation,True,fuel,steam_production,,,
 core_ferc1__yearly_operating_expenses_sched320,steam_expenses_steam_power_generation,True,non_fuel_operation,steam_production,,,