SQL(alchemy) storage engine

hax_debian.sh

@@ -10,7 +10,6 @@ fpm -s python \
     -a all \
     -m "Alistair Lynn <[email protected]>" \
     --deb-compression xz \
-    --deb-suggests python3-sqlalchemy \
     --deb-suggests python3-psycopg2 \
     -d "python3 (>= 3.5)" \
     -d python3-pkg-resources \

jacquard/cli.py

@@ -97,6 +97,10 @@ def main(args=sys.argv[1:], config=None):
 
     logging.basicConfig(level=options.log_level)
 
+    if options.log_level == logging.DEBUG:
+        # Explicitly enable sqlalchemy debugging
+        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+
     if options.func is None:
         parser.print_help()
         return

jacquard/storage/db.py

@@ -0,0 +1,108 @@
+import sqlalchemy
+import sqlalchemy.orm
+import sqlalchemy.sql.expression
+import sqlalchemy.ext.declarative
+
+from .base import StorageEngine
+
+_METADATA = sqlalchemy.MetaData()
+
+_Base = sqlalchemy.ext.declarative.declarative_base(metadata=_METADATA)
+
+
+class _Entry(_Base):
+    __tablename__ = 'entries'
+
+    id = sqlalchemy.Column(
+        sqlalchemy.Integer(),
+        primary_key=True,
+    )
+
+    key = sqlalchemy.Column(
+        sqlalchemy.String(length=200),
+        nullable=False,
+    )
+
+    value = sqlalchemy.Column(
+        sqlalchemy.Text(),
+        nullable=True,
+    )
+
+    created = sqlalchemy.Column(
+        sqlalchemy.DateTime(timezone=True),
+        nullable=False,
+        server_default=sqlalchemy.sql.expression.func.now(),
+    )
+
+    __table_args__ = (
+        sqlalchemy.Index(
+            'entries_by_key',
+            key,
+            id,
+        ),
+    )
+
+
+class DBStore(StorageEngine):
+    def __init__(self, connection_string):
+        self.engine = sqlalchemy.create_engine(
+            connection_string,
+            isolation_level='SERIALIZABLE',
+        )
+        _METADATA.create_all(bind=self.engine)
+        self.session = sqlalchemy.orm.Session(self.engine)
+
+    def begin(self):
+        pass
+
+    # TODO: Investigate `begin_read_only` with a lower isolation
+
+    def commit(self, changes, deletions):
+        self.session.add_all(
+            _Entry(key=key, value=value)
+            for key, value in changes.items()
+        )
+
+        self.session.add_all(
+            _Entry(key=key, value=None)
+            for key in deletions
+        )
+
+        # TODO: Intercept the need for Retry here since we're running with the
+        # SERIALIZABLE isolation level.
+        self.session.commit()
+
+    def rollback(self):
+        self.session.rollback()
+
+    def get(self, key):
+        node = self.session.query(
+            _Entry,
+        ).filter_by(
+            key=key,
+        ).order_by(
+            sqlalchemy.desc(_Entry.id),
+        ).first()
+
+        if node is None:
+            return None
+
+        return node.value
+
+    def keys(self):
+        # This is a slightly fiddly top-n-per-group situation, but that's OK:
+        # essentially we want to exclude any keys whose latest value is null.
+
+        later_entry = sqlalchemy.orm.aliased(_Entry)
+
+        query = self.session.query(
+            _Entry.key,
+        ).filter(
+            _Entry.value != None,  # noqa: E711
+            ~self.session.query(later_entry).filter(
+                later_entry.key == _Entry.key,
+                later_entry.id > _Entry.id,
+            ).exists(),
+        )
+
+        return [x for (x,) in query]

jacquard/storage/tests/test_db.py

@@ -0,0 +1,9 @@
+import unittest
+
+from jacquard.storage.db import DBStore
+from jacquard.storage.testing_utils import StorageGauntlet
+
+
+class DBStorageGauntlet(StorageGauntlet, unittest.TestCase):
+    def open_storage(self):
+        return DBStore('sqlite://')

setup.py

@@ -106,6 +106,7 @@ def debian_etc_hack(root):
             'redis = jacquard.storage.redis:RedisStore',
             'redis-cloned = jacquard.storage.cloned_redis:ClonedRedisStore',
             'file = jacquard.storage.file:FileStore',
+            'db = jacquard.storage.db:DBStore',
         ),
         'jacquard.commands': (
             'storage-dump = jacquard.storage.commands:StorageDump',