From ccec18e5668bf60b69b0f56b15eb63007e4bd110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannes=20H=C3=B6ke?= Date: Thu, 2 May 2024 10:17:29 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=91=BD=EF=B8=8F=20Update=20fixed=20wi?= =?UTF-8?q?dths=20for=20parsing=20stations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scraper/src/loadStations.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scraper/src/loadStations.js b/scraper/src/loadStations.js index d0681a7..3741e84 100644 --- a/scraper/src/loadStations.js +++ b/scraper/src/loadStations.js @@ -21,8 +21,11 @@ module.exports = async function loadStations (baseMinYear) { 'https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/KL_Tageswerte_Beschreibung_Stationen.txt'; const raw = (await got(url, { encoding: 'latin1' })).body; return parseFixedWidth(raw, { - skip: 3, - widths: [5, 9, 9, 15, 12, 10, 42, 22], + // The format was changed sometime between 2024-04-15 and 2024-05-01 + //skip: 3, + //widths: [5, 9, 9, 15, 12, 10, 42, 22], + skip: 2, + widths: [21, 9, 9, 14, 12, 10, 81, 843], names: ['id', 'from', 'to', 'altitude', 'lat', 'lon', 'name', 'state'], }) .map(station => ({ From 6aa48c349b33767532f4f3683686adbd61e82278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannes=20H=C3=B6ke?= Date: Thu, 2 May 2024 10:22:41 +0200 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20Raise=20error=20if=20parsed?= =?UTF-8?q?=20station=20list=20is=20empty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scraper/src/loadStations.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scraper/src/loadStations.js b/scraper/src/loadStations.js index 3741e84..fbbf935 100644 --- a/scraper/src/loadStations.js +++ b/scraper/src/loadStations.js @@ -20,7 +20,7 @@ module.exports = async function loadStations (baseMinYear) { const url = 'https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/KL_Tageswerte_Beschreibung_Stationen.txt'; const raw = (await got(url, { encoding: 'latin1' })).body; - return parseFixedWidth(raw, { + const stations = parseFixedWidth(raw, { // The format was changed sometime between 2024-04-15 and 2024-05-01 //skip: 3, //widths: [5, 9, 9, 15, 12, 10, 42, 22], @@ -40,6 +40,11 @@ module.exports = async function loadStations (baseMinYear) { .filter(station => !STATION_BLACKLIST.includes(station.name)) .filter(d => dayjs(d.from).year() <= baseMinYear && dayjs().diff(d.to, 'day') < 5) .sort((a, b) => (a.slug > b.slug ? 1 : b.slug > a.slug ? -1 : 0)); + + if (stations.length === 0) { + throw new Error('No stations found'); + } + return stations; }; function parseFixedWidth (data, { skip = 0, widths = [], names = [], trim = true }) {