Skip to content

Commit

Permalink
#1728: add pass_row as configurable parameter for field_update
Browse files Browse the repository at this point in the history
# Description
As described in #1728, pass_row parameter is now added as an editable parameter for the field_update step. This allows us to use row data inside custom python function.

# Tests
Ensured the newly added unit test ` test_step_field_update_with_function_and_pass_row_true` runs without fail and also that all the previous test cases run and pass too.
  • Loading branch information
ebAbhay committed Jan 22, 2025
1 parent 6b72909 commit c84590d
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 1 deletion.
28 changes: 28 additions & 0 deletions frictionless/steps/field/__spec__/test_field_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,31 @@ def test_step_field_update_referenced_as_foreign_key():
"reference": {"fields": ["pkey"], "resource": "resource1"},
}
]


def test_step_field_update_with_function_and_pass_row_true():
source = TableResource(path="data/transform.csv")

def add_country_to_id(value, row):
return str(value) + " " + row["name"]

pipeline = Pipeline(
steps=[
steps.field_update(
name="id", function=add_country_to_id,
descriptor={"type": "string"}, pass_row=True)
],
)
target = source.transform(pipeline)
assert target.schema.to_descriptor() == {
"fields": [
{"name": "id", "type": "string"},
{"name": "name", "type": "string"},
{"name": "population", "type": "integer"},
]
}
assert target.read_rows() == [
{"id": "1 germany", "name": "germany", "population": 83},
{"id": "2 france", "name": "france", "population": 66},
{"id": "3 spain", "name": "spain", "population": 47},
]
8 changes: 7 additions & 1 deletion frictionless/steps/field/field_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,16 @@ class field_update(Step):
A descriptor for the field to set the metadata.
"""

pass_row: bool = False
"""
Pass the entire row as a parameter for the transformation function when True.
"""

# Transform

def transform_resource(self, resource: Resource):
function = self.function
pass_row = False
pass_row = self.pass_row
table = resource.to_petl() # type: ignore
descriptor = deepcopy(self.descriptor) or {}
new_name = descriptor.get("name")
Expand Down Expand Up @@ -106,5 +111,6 @@ def transform_resource(self, resource: Resource):
"value": {},
"formula": {"type": "string"},
"descriptor": {"type": "object"},
"pass_row": {"type": "boolean"},
},
}

0 comments on commit c84590d

Please sign in to comment.