Make csv format compatible with Unfolded Studio import (#52)

Unfolded/core/processing/csv_field_value_converter.py

@@ -21,7 +21,10 @@
 
 
 class CsvFieldValueConverter(QgsVectorFileWriter.FieldValueConverter):
-    """ Converts boolean fields to string fields containig true, false or empty string """
+    """
+    Converts boolean fields to string fields containing true, false or empty string.
+    Also cast first field values to string to avoid import problems on Unfolded Studio.
+    """
 
     def __init__(self, layer: QgsVectorLayer):
         QgsVectorFileWriter.FieldValueConverter.__init__(self)
@@ -38,6 +41,8 @@ def convert(self, field_idx, value):
 
     def fieldDefinition(self, field):
         idx = self.layer.fields().indexFromName(field.name())
-        if idx in self.bool_field_idxs:
+
+        # Cast the first cell value to string
+        if idx in self.bool_field_idxs or idx == 0:
             return QgsField(field.name(), QVariant.String)
         return self.layer.fields()[idx]

Unfolded/core/processing/layer2dataset.py

@@ -202,7 +202,7 @@ def _save_layer_to_file(layer: QgsVectorLayer, output_path: Path) -> Path:
         options = QgsVectorFileWriter.SaveVectorOptions()
         options.driverName = "csv"
         options.fileEncoding = "utf-8"
-        options.layerOptions = ["SEPARATOR=TAB", "STRING_QUOTING=ALWAYS"]
+        options.layerOptions = ["SEPARATOR=TAB", "STRING_QUOTING=IF_AMBIGUOUS"]
         options.fieldValueConverter = converter
 
         # noinspection PyCallByClass

Unfolded/test/data/harbours.csv

@@ -1,17 +1,17 @@
-"fid"	"nimi"	"tonnia_vienti"	"tonnia_tuonti"	"date"	"datetime"	"bool"	"longitude"	"latitude"
-"1"	"Eurajoki"	"85920"	"213262"	"2021/01/26"	"2021/01/26 10:05:56"	"true"	21.48781	61.24682
-"2"	"Forby"	"10516"	"223134"	"2021/01/26"	"2021/01/26 10:05:29"	"false"	22.86569	60.09882
-"3"	"HaminaKotka"	"10386150"	"4709429"	"2021/01/06"	"2021/01/26 10:05:28"	""	26.94052	60.46855
-"4"	"Hanko"	"2592483"	"2160444"	"2021/01/26"	"2021/01/11 00:00:00"	"false"	22.95667	59.82226
-"5"	"Helsinki"	"7377509"	"7193129"	"2021/01/26"	"2021/01/13 00:00:00"	"true"	25.17551	60.22284
-"6"	"Inkoo"	"864003"	"1012706"	"2021/01/26"	"2021/01/07 12:50:00"	"true"	24.00769	60.04306
-"7"	"Kemio"	"18505"	"34066"	"2021/01/26"	"2021/01/26 10:05:34"	"true"	22.74995	60.1712
-"8"	"Lappeenranta"	"37047"	"279423"	"2021/01/26"	"2021/01/21 00:00:00"	"true"	28.32973	61.05645
-"9"	"Naantali"	"1542843"	"4139181"	"2021/01/26"	"2021/02/11 00:00:00"	"false"	22.0923	60.44687
-"10"	"Parainen"	"144023"	"331196"	"2021/01/26"	"2021/01/26 10:05:54"	"true"	22.30066	60.29697
-"11"	"Pori"	"1934234"	"1553107"	"2021/01/26"	"2021/01/26 10:05:53"	"false"	21.48514	61.59316
-"12"	"Rauma"	"4162522"	"1588063"	"2021/01/26"	"2021/01/26 10:05:55"	"false"	21.45215	61.12832
-"13"	"Skoldvik"	"8557615"	"12873862"	"2021/01/26"	"2021/01/26 10:05:55"	"false"	25.5374	60.2941
-"14"	"Taalintehdas"	"4404"	"4620"	"2021/01/26"	"2021/01/26 10:05:56"	"false"	22.50803	60.0204
-"15"	"Turku"	"1122041"	"994114"	"2021/01/26"	"2021/01/26 10:05:56"	"true"	22.22377	60.43468
-"16"	"Uusikaupunki"	"1454930"	"1024675"	"2021/01/26"	"2021/01/26 10:05:58"	"false"	21.38951	60.797
+fid	nimi	tonnia_vienti	tonnia_tuonti	date	datetime	bool	longitude	latitude
+"1"	Eurajoki	"85920"	"213262"	2021/01/26	2021/01/26 10:05:56	true	21.48781	61.24682
+"2"	Forby	"10516"	"223134"	2021/01/26	2021/01/26 10:05:29	false	22.86569	60.09882
+"3"	HaminaKotka	"10386150"	"4709429"	2021/01/06	2021/01/26 10:05:28		26.94052	60.46855
+"4"	Hanko	"2592483"	"2160444"	2021/01/26	2021/01/11 00:00:00	false	22.95667	59.82226
+"5"	Helsinki	"7377509"	"7193129"	2021/01/26	2021/01/13 00:00:00	true	25.17551	60.22284
+"6"	Inkoo	"864003"	"1012706"	2021/01/26	2021/01/07 12:50:00	true	24.00769	60.04306
+"7"	Kemio	"18505"	"34066"	2021/01/26	2021/01/26 10:05:34	true	22.74995	60.1712
+"8"	Lappeenranta	"37047"	"279423"	2021/01/26	2021/01/21 00:00:00	true	28.32973	61.05645
+"9"	Naantali	"1542843"	"4139181"	2021/01/26	2021/02/11 00:00:00	false	22.0923	60.44687
+"10"	Parainen	"144023"	"331196"	2021/01/26	2021/01/26 10:05:54	true	22.30066	60.29697
+"11"	Pori	"1934234"	"1553107"	2021/01/26	2021/01/26 10:05:53	false	21.48514	61.59316
+"12"	Rauma	"4162522"	"1588063"	2021/01/26	2021/01/26 10:05:55	false	21.45215	61.12832
+"13"	Skoldvik	"8557615"	"12873862"	2021/01/26	2021/01/26 10:05:55	false	25.5374	60.2941
+"14"	Taalintehdas	"4404"	"4620"	2021/01/26	2021/01/26 10:05:56	false	22.50803	60.0204
+"15"	Turku	"1122041"	"994114"	2021/01/26	2021/01/26 10:05:56	true	22.22377	60.43468
+"16"	Uusikaupunki	"1454930"	"1024675"	2021/01/26	2021/01/26 10:05:58	false	21.38951	60.797

Unfolded/test/test_config_creator.py

@@ -74,6 +74,9 @@ def test_map_config_creation_w_simple_points(config_creator, simple_harbour_poin
 
 def test_map_config_creation_with_unfolded_format(config_creator, simple_harbour_points):
     with config_creator as cf:
+        time_zone = time.strftime('%Z%z')
+        excpected_map_config = get_map_config('harbours_config_with_unfolded_datasets.json')
+        excpected_map_config.info.created_at = excpected_map_config.info.created_at.replace("EET+0200", time_zone)
         cf.add_layer(uuid.UUID('7d193484-21a7-47f4-8cbc-497474a39b64'), simple_harbour_points,
                      QColor.fromRgb(0, 92, 255), True)
         cf._start_config_creation()
@@ -82,6 +85,9 @@ def test_map_config_creation_with_unfolded_format(config_creator, simple_harbour
         assert cf.created_configuration_path.name == 'unfolded_nabzfz.zip'
         with ZipFile(cf.created_configuration_path, 'r') as zip_file:
             assert [n for n in zip_file.namelist()] == ['config.json', 'harbours.csv']
+        map_config = get_loaded_map_config(cf.created_configuration_path)
+        assert map_config.to_dict() == excpected_map_config.to_dict()
+
     assert not cf._temp_dir.exists()