2024 data

.github/workflows/ci.yml

@@ -17,13 +17,15 @@ jobs:
       - name: Install Ubuntu dependencies
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -y -qq --no-install-recommends python3-gdal python3-pip python3-pytest unzip
+          sudo apt-get install -y -qq --no-install-recommends libgdal-dev unzip
 
       - name: Install Python dependencies
-        run: pip install -r requirements.txt
+        run: |
+          pip install -r requirements.txt
+          pip install pytest
 
       - name: Run tests
-        run: pytest-3
+        run: pytest
 
       - name: Create output
         run: |

.gitignore

@@ -1 +1,2 @@
-*/__pycache__
+*/__pycache__
+venv-tiger

README.md

@@ -15,8 +15,11 @@ Replace '2024' with the current year throughout.
 
         ```bash
         # Ubuntu:
-        sudo apt-get install python3-gdal python3-pip unzip
-        pip3 install -r requirements.txt
+        sudo apt-get install python3
+
+        python3 -m venv venv-tiger
+        . venv-tiger/bin/activate
+        pip install -r requirements.txt
         ```
 
   2. Get the TIGER 2024 data. You will need the EDGES files
@@ -35,14 +38,16 @@ Replace '2024' with the current year throughout.
 
   3. Convert the data into CSV files. Adjust the file paths in the scripts as needed
 
-        ```bash
+         ```bash
         ./convert.sh <input-path> <output-path>
+        cd output-path
+        ./patch.sh
          ```
 
   4. Maybe: package the created files
 
-        ```bash
-        tar -czf tiger2024-nominatim-preprocessed.csv.tar.gz tiger
+         ```bash
+        tar -czf tiger2024-nominatim-preprocessed.csv.tar.gz *.csv
          ```
 
 

lib/parse.py

@@ -6,10 +6,7 @@
 import json
 import re
 
-try:
-    from osgeo import ogr
-except ImportError:
-    import ogr
+from osgeo import ogr
 
 # https://www.census.gov/geo/reference/codes/cou.html
 # tiger_county_fips.json was generated from the following:

lib/project.py

@@ -2,10 +2,9 @@
 Deal with coordinate system transformations/projections
 """
 
-try:
-    from osgeo import osr
-except ImportError:
-    import osr
+from osgeo import osr
+
+osr.DontUseExceptions()
 
 # Same as the contents of the *_edges.prj files
 PROJCS_WKT = \
@@ -27,4 +26,4 @@
 def unproject(point):
     """Covert point to WGS84"""
     projected = transformer.TransformPoint(point[0], point[1])
-    return (projected[0], projected[1])
+    return (round(projected[0], 6), round(projected[1], 6))

patch.sh

@@ -0,0 +1,5 @@
+#/bin/bash
+
+# The 1..45 line is way off
+grep -v '45;1;all;Woodfall Rd;Middlesex;MA;02478' 25017.csv > 25017.csv.tmp
+mv 25017.csv.tmp 25017.csv

requirements.txt

@@ -0,0 +1 @@
+gdal==3.8.4

tests/test_project.py

@@ -1,4 +1,5 @@
 from lib.project import unproject
 
 def test_unproject():
+    # This test fails on my MacOS, no idea why, must be my local python setup
     assert(unproject([-76.521714, 36.330247])) == (36.330247, -76.521714)

tiger_address_convert.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 """
 Tiger road data to OSM conversion script