Merge branch 'master' into peter

doc/manual.adoc

@@ -50,6 +50,85 @@ version on start.  newfol does not accept schema files that have a newer version
 than it currently supports, but it ignores directives it does not understand,
 and so can be manually downgraded by editing the `fmt` directive.
 
+Any record beginning with a `#` is ignored as a comment and preserved on
+upgrades.  Extra fields in a record are ignored.
+
+The standard rules for formatting colon-separated values apply.  In other words,
+the format is defined as identical to the `text/csv` format specified in RFC
+4180, except that the delimiter is a colon instead of a comma and that the line
+break is a single linefeed instead of a CRLF pair.
+
+==== Format Directives (`fmt`)
+
+Format directives define the file as a schema file or configuration file as
+specified in <<_schema_file>> and <<_configuration_file>>.  They must appear
+only at the very beginning of a schema or configuration file.  In particular, it
+is not allowed for them to be preceded by a byte order mark.
+
+==== Table Definition Directives (`def`)
+
+A table definition directive consists of a record containing the values `def`
+and the table name.  A table must be defined before any field definitions can
+occur for that table.
+
+==== Field Definition Directives (`fld` and `dsc`)
+
+Field definitions come in two types.  The first, `fld`, provides a mapping for
+imported data, and `dsc` does not.  They are otherwise identical.
+
+A field definition directive consists of a record containing the values `fld` or
+`dsc`, the name of the table, the field number for imported data (or empty for
+`dsc`), the field number for use in newfol, an optional Python 3 expression, and
+a description.
+
+The Python 3 expression is evaluated and placed in the field when creating a new
+record.  This can be used, for example, to automatically insert the date into a
+particular field.  This requires that code execution has been allowed by an
+`exe` directive.
+
+==== Execution Directives (`exe` and `execute`)
+
+Execution directives specify whether Python code will be run from the schema
+file.  Such a directive consists of a record containing either `exe` or
+`execute` and then either `yes` or `no`.
+
+Unlike other directives, where the schema file overrides the configuration file,
+if any execution directive is set to `no`, execution is disabled.  This is a
+security feature to allow reading an untrusted database.  Execution is also
+disabled if more than one execution directive is seen in either the schema or
+configuration file.
+
+==== Key Field Directives (`key`)
+
+Key field directives indicate the fields to display in the _Browse All Records_
+view.  By default, this value is 0 (display the first field).
+
+==== Column Directives (`col`)
+
+Column directives indicate how many columns to use to display fields.  A column
+directive consists of a record with the values `col` and an integer number of
+columns.
+
+==== Hook Directives (`txn` and `hook`)
+
+Hook directives indicate a series of hooks to be run on each load or save
+operation.  Several different hooks are available.  The default set of hooks is
+`hash`.  This provides an SHA-2 hash of the data in the `dtb.checksum` file, and
+cannot be disabled, only replaced with a specific algorithm.
+
+By default, the hash is `sha256` on 32-bit systems and `sha512` on 64-bit
+systems; `sha384` is also available.  If a specific algorithm is specified, it
+overrides the `hash` default.  Older versions of newfol only supported the
+`sha256` algorithm.
+
+The other available hook is `git`.  This hook performs a commit every time the
+file is saved, and a git checkout of the `dtb` file before every load.  This
+allows a user to keep a history of the data and to easily back it up to another
+location.
+
+A hook directive consists of a record starting with the value `txn`, and
+followed by a list of hook names.  `hook` may be used as a synonym for `txn`.
+
 === Configuration File
 
 Configuration files are identical to schema files except that instead of the

lib/newfol/database.py

@@ -169,7 +169,7 @@ class Schema:
         self._nfields = nfields
         self._mapping = {}
         self._keyfields = []
-        self._exe_ok = False
+        self._exe_ok = None
         self._imports = []
         self._layout = list(range(nfields))
         self._cheatsheet = None
@@ -185,7 +185,7 @@ class Schema:
         return self._txntype
 
     def execution_allowed(self):
-        return self._exe_ok
+        return self._exe_ok or False
 
     def mapping(self):
         return self._mapping
@@ -329,7 +329,7 @@ class Schema:
         mapping = {}
         keyfields = []
         nfields = 0
-        exe_ok = False
+        exe_ok = None
         exe_seen = False
         for i in recs:
             if len(i.fields) == 0:
@@ -370,6 +370,8 @@ class Schema:
                 exe_seen = True
                 if i.fields[1].lower() in ("yes", "oui", "sí"):
                     exe_ok = True
+                else:
+                    exe_ok = False
             elif rectype in ['imp', 'import']:
                 self._imports = filter(lambda x: x, i.fields[1:])
             elif rectype in ['pvu', 'preview']:
@@ -381,8 +383,14 @@ class Schema:
                 else:
                     self._cheatsheet = path + "/" + cheatsheet
             elif rectype in ['col', 'column']:
-                self._columns = int(i.fields[1])
+                try:
-            elif rectype in ['txn', 'transaction']:
+                    self._columns = int(i.fields[1])
+                    if self._columns <= 0:
+                        raise ValueError
+                except ValueError:
+                    raise newfol.exception.SchemaError("has invalid " +
+                                                       "number of columns")
+            elif rectype in ['txn', 'transaction', 'hook']:
                 self._txntype = i.fields[1:]
             elif rectype in ['dpy', 'display']:
                 self._layout = list(map(fix_up_layout, i.fields[1:]))
@@ -400,7 +408,10 @@ class Schema:
                     self._keys[i.fields[1]] = None
         self._mapping = mapping
         self._keyfields = keyfields
-        self._exe_ok = exe_ok
+        if self._exe_ok is None:
+            self._exe_ok = exe_ok
+        elif exe_ok is False:
+            self._exe_ok = False
 
 
 class Database:

lib/newfol/exception.py

@@ -15,7 +15,7 @@ class LocaleError(NewfolError):
 
 
 class SchemaError(NewfolError):
-    def __init__(msg):
+    def __init__(self, msg):
         super().__init__("Schema file %s" % msg)
 
 
@@ -28,7 +28,7 @@ class FilemanipError(Exception):
 
 
 class CorruptFileError(FilemanipError):
-    def __init__(type, msg):
+    def __init__(self, type, msg):
         super().__init__("%s file is corrupt (%s)" % (type, msg))
 
 

lib/newfol/filemanip.py

@@ -61,7 +61,7 @@ class Record:
         return self._version
 
 
-class TransactionStore:
+class Hook:
     def prepare_open(self, filename, mode):
         pass
 
@@ -87,7 +87,7 @@ class TransactionStore:
         pass
 
 
-class StackingTransactionStore(TransactionStore):
+class StackingHook(Hook):
     def __init__(self, stores):
         self._stores = stores
 
@@ -124,7 +124,7 @@ class StackingTransactionStore(TransactionStore):
             store.commit_store(rec)
 
 
-class GitTransactionStore(TransactionStore):
+class GitHook(Hook):
 
     class DirectoryChanger:
 
@@ -198,7 +198,7 @@ class GitTransactionStore(TransactionStore):
                 self._call_git("commit", "-m", message)
 
 
-class HashTransactionStore(TransactionStore):
+class HashHook(Hook):
     def __init__(self, hashname="sha256", options=None):
         self._filename = None
         self._mode = None
@@ -230,7 +230,7 @@ class HashTransactionStore(TransactionStore):
             return choice1
         if choice2 is not None:
             return choice2
-        return "sha512" if HashTransactionStore._bitness() == 64 else "sha256"
+        return "sha512" if HashHook._bitness() == 64 else "sha256"
 
     def prepare_open(self, filename, mode):
         self._filename = filename
@@ -578,11 +578,11 @@ class FileStorage:
     @staticmethod
     def _get_transaction_types():
         return {
-            "git": GitTransactionStore,
+            "git": GitHook,
-            "sha256": lambda *a, **k: HashTransactionStore("sha256", *a, **k),
+            "sha256": lambda *a, **k: HashHook("sha256", *a, **k),
-            "sha384": lambda *a, **k: HashTransactionStore("sha384", *a, **k),
+            "sha384": lambda *a, **k: HashHook("sha384", *a, **k),
-            "sha512": lambda *a, **k: HashTransactionStore("sha512", *a, **k),
+            "sha512": lambda *a, **k: HashHook("sha512", *a, **k),
-            "hash": lambda *a, **k: HashTransactionStore(None, *a, **k),
+            "hash": lambda *a, **k: HashHook(None, *a, **k),
         }
 
     @staticmethod
@@ -593,13 +593,13 @@ class FileStorage:
     def _make_transaction_store(items, options):
         txntypes = FileStorage._get_transaction_types()
         if items is None or items == "":
-            return TransactionStore()
+            return Hook()
         elif isinstance(items, str):
             return txntypes[items](options)
         else:
             stores = [FileStorage._make_transaction_store(i, options)
                       for i in items]
-            return StackingTransactionStore(stores)
+            return StackingHook(stores)
 
     def store(self, records):
         """Store the records."""

test/testdatabase.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+from newfol.exception import SchemaError
 from newfol.database import DatabaseVersion, Database, Schema, Singleton
 from newfol.filemanip import Record
 import tempfile
@@ -59,60 +60,56 @@ class TestDatabaseAccessors(unittest.TestCase):
                          DatabaseVersion.preferred().serialization())
 
 
-class TestDatabaseIntegrity(unittest.TestCase):
+class TemporaryDatabase:
-    def create_temp_db(self):
+    def __init__(self, schema_contents=""):
-        ddir = tempfile.TemporaryDirectory()
+        self.ddir = tempfile.TemporaryDirectory()
-        with open(ddir.name + "/schema", "w") as fp:
+        with open(self.ddir.name + "/schema", "w") as fp:
-            fp.write("fmt:0:newfol schema file:\ntxn:git\n")
+            fp.write("fmt:3:newfol schema file:\n" + schema_contents)
-        db = Database.load(ddir.name)
-        return (ddir, db)
 
+    @property
+    def db(self):
+        return Database.load(self.ddir.name)
+
+    def __del__(self):
+        self.ddir.cleanup()
+
+
+class TestDatabaseIntegrity(unittest.TestCase):
     def test_version(self):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase()
-        self.assertEqual(Database.read_version(ddir.name),
+        self.assertEqual(Database.read_version(tdb.ddir.name),
                          DatabaseVersion())
-        db.store()
+        tdb.db.store()
-        db.upgrade()
+        tdb.db.upgrade()
-        self.assertEqual(Database.read_version(ddir.name),
+        self.assertEqual(Database.read_version(tdb.ddir.name),
                          DatabaseVersion.preferred())
-        ddir.cleanup()
 
     def test_validate(self):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase()
-        db.store()
+        tdb.db.store()
-        db.validate()
+        tdb.db.validate()
-        ddir.cleanup()
 
     def test_validate_strict(self):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase()
-        db.store()
+        tdb.db.store()
-        db.validate(strict=True)
+        tdb.db.validate(strict=True)
-        ddir.cleanup()
 
     def test_repair_doesnt_raise(self):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase()
-        db.store()
+        tdb.db.store()
-        db.repair()
+        tdb.db.repair()
-        db.validate(strict=True)
+        tdb.db.validate(strict=True)
-        ddir.cleanup()
 
     def test_upgrade_records(self):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase()
-        db.store()
+        tdb.db.store()
-        db.upgrade_records()
+        tdb.db.upgrade_records()
-        ddir.cleanup()
 
 
 class TestDatabaseUpgrades(unittest.TestCase):
-    def create_temp_db(self):
-        ddir = tempfile.TemporaryDirectory()
-        with open(ddir.name + "/schema", "w") as fp:
-            fp.write("fmt:0:newfol schema file:\ntxn:git\n")
-        db = Database.load(ddir.name)
-        return (ddir, db)
-
     def do_upgrade_test(self, version, pattern):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase("txn:git\n")
+        ddir, db = tdb.ddir, tdb.db
         if not isinstance(version, DatabaseVersion):
             version = DatabaseVersion(version)
         self.assertEqual(Database.read_version(ddir.name),
@@ -141,56 +138,84 @@ class TestDatabaseUpgrades(unittest.TestCase):
 
 class TestMultipleTransactions(unittest.TestCase):
     def test_multiple_types(self):
-        ddir = tempfile.TemporaryDirectory()
+        tdb = TemporaryDatabase("txn:git:hash\n")
-        with open(ddir.name + "/schema", "w") as fp:
+        ddir = tdb.ddir
-            fp.write("fmt:0:newfol schema file:\ntxn:git:hash\n")
         db = Database.load(ddir.name)
         db.records()[:] = [Record([1, 2, 3])]
         db.store()
         self.assertEqual(set(db.schema().transaction_types()),
                          set(["git", "hash"]))
-        ddir.cleanup()
 
 
 class TestExtraSchemaConfig(unittest.TestCase):
     def test_existing_config_file(self):
-        ddir1 = tempfile.TemporaryDirectory()
+        tdb = TemporaryDatabase()
         ddir2 = tempfile.TemporaryDirectory()
         config = "%s/config" % ddir2.name
-        with open(ddir1.name + "/schema", "w") as fp:
-            fp.write("fmt:3:newfol schema file:\n")
         with open(config, "w") as fp:
             fp.write("fmt:3:newfol config file:\ntxn:git:hash\n")
-        db = Database.load(ddir1.name, extra_config=[config])
+        db = Database.load(tdb.ddir.name, extra_config=[config])
         db.records()[:] = [Record([1, 2, 3])]
         db.store()
         self.assertEqual(set(db.schema().transaction_types()),
                          set(["git", "hash"]))
-        ddir1.cleanup()
         ddir2.cleanup()
 
     def test_missing_config_file(self):
-        ddir1 = tempfile.TemporaryDirectory()
+        tdb = TemporaryDatabase()
-        config = "%s/config" % ddir1.name
+        config = "%s/config" % tdb.ddir.name
-        with open(ddir1.name + "/schema", "w") as fp:
+        db = Database.load(tdb.ddir.name, extra_config=[config])
-            fp.write("fmt:3:newfol schema file:\n")
-        db = Database.load(ddir1.name, extra_config=[config])
         db.records()[:] = [Record([1, 2, 3])]
         db.store()
         # Ensure no exception is raised.
-        ddir1.cleanup()
+
+
+class TestSchemaColumns(unittest.TestCase):
+    def check_invalid(self, value):
+        with self.assertRaises(SchemaError):
+            tdb = TemporaryDatabase("col:%s\n" % value)
+            tdb.db
+
+    def test_non_integral(self):
+        self.check_invalid(3.5)
+
+    def test_negative(self):
+        self.check_invalid(-1)
+
+    def test_zero(self):
+        self.check_invalid(0)
+
+
+class TestExecutionAllowed(unittest.TestCase):
+    def do_test(self, expected, schema, configv):
+        tdb = TemporaryDatabase(schema)
+        ddir2 = tempfile.TemporaryDirectory()
+        config = "%s/config" % ddir2.name
+        with open(config, "w") as fp:
+            fp.write("fmt:3:newfol config file:\n" + configv)
+        db = Database.load(tdb.ddir.name, extra_config=[config])
+        db.records()[:] = [Record([1, 2, 3])]
+        db.store()
+        self.assertEqual(db.schema().execution_allowed(), expected)
+        ddir2.cleanup()
+
+    def test_true_if_only_true_schema(self):
+        self.do_test(True, "exe:yes\n", "")
+
+    def test_true_if_only_true_config(self):
+        self.do_test(True, "", "exe:yes\n")
+
+    def test_false_if_schema_false(self):
+        self.do_test(False, "exe:no\n", "exe:yes\n")
+
+    def test_false_if_config_false(self):
+        self.do_test(False, "exe:yes\n", "exe:no\n")
 
 
 class TestDatabaseFiltering(unittest.TestCase):
-    def create_temp_db(self):
-        ddir = tempfile.TemporaryDirectory()
-        with open(ddir.name + "/schema", "w") as fp:
-            fp.write("fmt:0:newfol schema file:\ntxn:git\n")
-        db = Database.load(ddir.name)
-        return (ddir, db)
-
     def test_filtering(self):
-        ddir, db = self.create_temp_db()
+        tdb = TemporaryDatabase("txn:git\n")
+        db = tdb.db
         records = [
             Record(["a", "b", "c"]),
             Record([1, 2, 3]),
@@ -208,7 +233,6 @@ class TestDatabaseFiltering(unittest.TestCase):
         selected = db.records(has_only_numbers)
         self.assertEqual(type(selected), list)
         self.assertEqual(set(selected), set(records[1:]))
-        ddir.cleanup()
 
 
 class TestSingleton(unittest.TestCase):

test/testfilemanip.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from newfol.filemanip import Record, FileStorage, HashTransactionStore
+from newfol.filemanip import Record, FileStorage, HashHook
 import hashlib
 import newfol.exception
 import tempfile
@@ -140,7 +140,7 @@ class SHA256TransactionTest(unittest.TestCase):
             v = self.generate_len_and_hash(hashlib.sha256, k)
             with open(temp, "w") as wfp:
                 wfp.write(k)
-            self.assertEqual(HashTransactionStore._hash_file("sha256", temp),
+            self.assertEqual(HashHook._hash_file("sha256", temp),
                              "sha256:" + v)
         tempdir.cleanup()
 
@@ -210,7 +210,7 @@ class SHA256TransactionTest(unittest.TestCase):
         x.fs.store([Record(["", ""])])
         with open(x.tempfile + ".checksum", "r") as fp:
             data = fp.read()
-        bitness = HashTransactionStore._bitness()
+        bitness = HashHook._bitness()
         self.assertIn(bitness, [32, 64])
         self.assertTrue(data.startswith("sha512" if bitness == 64 else
                                         "sha256"))