Sets asc order on event request and start_date is now - 30 days

packages/flare/bin/cron_job_ingest_events.py

@@ -8,8 +8,9 @@
 import json
 import os
 
-from datetime import date
 from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
 from typing import Any
 from typing import Iterator
 from typing import Optional
@@ -204,13 +205,13 @@ def get_next(kvstore: KVStoreCollections, tenant_id: int) -> Optional[str]:
     )
 
 
-def get_start_date(kvstore: KVStoreCollections) -> Optional[date]:
+def get_start_date(kvstore: KVStoreCollections) -> Optional[datetime]:
     start_date = get_collection_value(
         kvstore=kvstore, key=CollectionKeys.START_DATE.value
     )
     if start_date:
         try:
-            return date.fromisoformat(start_date)
+            return datetime.fromisoformat(start_date)
         except Exception:
             pass
     return None
@@ -255,7 +256,7 @@ def save_last_ingested_tenant_id(kvstore: KVStoreCollections, tenant_id: int) ->
         save_collection_value(
             kvstore=kvstore,
             key=CollectionKeys.START_DATE.value,
-            value=date.today().isoformat(),
+            value=(datetime.now(timezone.utc) - timedelta(days=30)).isoformat(),
         )
 
     save_collection_value(

packages/flare/bin/flare.py

@@ -8,7 +8,9 @@
 import requests
 import time
 
-from datetime import date
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
 from logger import Logger
 from typing import Any
 from typing import Dict
@@ -55,7 +57,7 @@ def fetch_feed_events(
         self,
         *,
         next: Optional[str] = None,
-        start_date: Optional[date] = None,
+        start_date: Optional[datetime] = None,
         ingest_full_event_data: bool,
         severities: list[str],
         source_types: list[str],
@@ -81,17 +83,18 @@ def _fetch_event_feed_metadata(
         self,
         *,
         next: Optional[str] = None,
-        start_date: Optional[date] = None,
+        start_date: Optional[datetime] = None,
         severities: list[str],
         source_types: list[str],
     ) -> Iterator[requests.Response]:
         data: Dict[str, Any] = {
             "from": next if next else None,
+            "order": "asc",
             "filters": {
                 "materialized_at": {
                     "gte": start_date.isoformat()
                     if start_date
-                    else date.today().isoformat()
+                    else (datetime.now(timezone.utc) - timedelta(days=30)).isoformat()
                 },
             },
         }

packages/flare/tests/bin/test_ingest_events.py

@@ -6,8 +6,8 @@
 from conftest import FakeKVStoreCollections
 from conftest import FakeLogger
 from conftest import FakeStoragePasswords
-from datetime import date
 from datetime import datetime
+from datetime import timezone
 from freezegun import freeze_time
 
 
@@ -101,11 +101,11 @@ def test_get_start_date_expect_none(kvstore: FakeKVStoreCollections) -> None:
 
 @pytest.mark.parametrize(
     "kvstore",
-    [[(CollectionKeys.START_DATE.value, "2000-01-01")]],
+    [[(CollectionKeys.START_DATE.value, "2000-01-01T00:00:00+00:00")]],
     indirect=True,
 )
 def test_get_start_date_expect_date(kvstore: FakeKVStoreCollections) -> None:
-    assert get_start_date(kvstore) == date(2000, 1, 1)
+    assert get_start_date(kvstore) == datetime(2000, 1, 1, 0, 0, tzinfo=timezone.utc)
 
 
 def test_get_last_ingested_tenant_id_expect_none(
@@ -125,7 +125,7 @@ def test_get_last_ingested_tenant_id_expect_integer(
     assert get_last_ingested_tenant_id(kvstore=kvstore) == 11111
 
 
-@freeze_time("2000-01-01")
+@freeze_time("2000-01-01T00:00:00+00:00")
 def test_save_last_ingested_tenant_id_expect_new_tenant_id_and_new_start_date(
     kvstore: FakeKVStoreCollections,
 ) -> None:
@@ -135,7 +135,7 @@ def test_save_last_ingested_tenant_id_expect_new_tenant_id_and_new_start_date(
 
     save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=11111)
     assert kvstore[KV_COLLECTION_NAME].data.query() == [
-        {"_key": CollectionKeys.START_DATE.value, "value": "2000-01-01"},
+        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-02T00:00:00+00:00"},
         {"_key": CollectionKeys.LAST_INGESTED_TENANT_ID.value, "value": 11111},
     ]
 
@@ -144,23 +144,23 @@ def test_save_last_ingested_tenant_id_expect_new_tenant_id_and_new_start_date(
     "kvstore",
     [
         [
-            (CollectionKeys.START_DATE.value, "1999-12-12"),
+            (CollectionKeys.START_DATE.value, "1999-12-12T00:00:00+00:00"),
             (CollectionKeys.LAST_INGESTED_TENANT_ID.value, 11111),
         ]
     ],
     indirect=True,
 )
-@freeze_time("2000-01-01")
+@freeze_time("2000-01-01T00:00:00+00:00")
 def test_save_last_ingested_tenant_id_expect_updated_tenant_id_and_updated_start_date(
     kvstore: FakeKVStoreCollections,
 ) -> None:
     assert kvstore[KV_COLLECTION_NAME].data.query() == [
-        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-12"},
+        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-12T00:00:00+00:00"},
         {"_key": CollectionKeys.LAST_INGESTED_TENANT_ID.value, "value": 11111},
     ]
     save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=22222)
     assert kvstore[KV_COLLECTION_NAME].data.query() == [
-        {"_key": CollectionKeys.START_DATE.value, "value": "2000-01-01"},
+        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-02T00:00:00+00:00"},
         {"_key": CollectionKeys.LAST_INGESTED_TENANT_ID.value, "value": 22222},
     ]
 
@@ -311,6 +311,6 @@ def test_main_expect_normal_run(
         flare_api_cls=FakeFlareAPI,
     )
     assert logger.messages == [
-        "INFO: Fetching tenant_id=11111, next=None, start_date=FakeDate(2000, 1, 1)",
+        "INFO: Fetching tenant_id=11111, next=None, start_date=FakeDatetime(1999, 12, 2, 0, 0, tzinfo=datetime.timezone.utc)",
         "INFO: Fetched 2 events",
     ]