diff --git a/doc/manual.adoc b/doc/manual.adoc index c03e917..fc4cec4 100644 --- a/doc/manual.adoc +++ b/doc/manual.adoc @@ -50,6 +50,85 @@ version on start. newfol does not accept schema files that have a newer version than it currently supports, but it ignores directives it does not understand, and so can be manually downgraded by editing the `fmt` directive. +Any record beginning with a `#` is ignored as a comment and preserved on +upgrades. Extra fields in a record are ignored. + +The standard rules for formatting colon-separated values apply. In other words, +the format is defined as identical to the `text/csv` format specified in RFC +4180, except that the delimiter is a colon instead of a comma and that the line +break is a single linefeed instead of a CRLF pair. + +==== Format Directives (`fmt`) + +Format directives define the file as a schema file or configuration file as +specified in <<_schema_file>> and <<_configuration_file>>. They must appear +only at the very beginning of a schema or configuration file. In particular, it +is not allowed for them to be preceded by a byte order mark. + +==== Table Definition Directives (`def`) + +A table definition directive consists of a record containing the values `def` +and the table name. A table must be defined before any field definitions can +occur for that table. + +==== Field Definition Directives (`fld` and `dsc`) + +Field definitions come in two types. The first, `fld`, provides a mapping for +imported data, and `dsc` does not. They are otherwise identical. + +A field definition directive consists of a record containing the values `fld` or +`dsc`, the name of the table, the field number for imported data (or empty for +`dsc`), the field number for use in newfol, an optional Python 3 expression, and +a description. + +The Python 3 expression is evaluated and placed in the field when creating a new +record. This can be used, for example, to automatically insert the date into a +particular field. This requires that code execution has been allowed by an +`exe` directive. + +==== Execution Directives (`exe` and `execute`) + +Execution directives specify whether Python code will be run from the schema +file. Such a directive consists of a record containing either `exe` or +`execute` and then either `yes` or `no`. + +Unlike other directives, where the schema file overrides the configuration file, +if any execution directive is set to `no`, execution is disabled. This is a +security feature to allow reading an untrusted database. Execution is also +disabled if more than one execution directive is seen in either the schema or +configuration file. + +==== Key Field Directives (`key`) + +Key field directives indicate the fields to display in the _Browse All Records_ +view. By default, this value is 0 (display the first field). + +==== Column Directives (`col`) + +Column directives indicate how many columns to use to display fields. A column +directive consists of a record with the values `col` and an integer number of +columns. + +==== Hook Directives (`txn` and `hook`) + +Hook directives indicate a series of hooks to be run on each load or save +operation. Several different hooks are available. The default set of hooks is +`hash`. This provides an SHA-2 hash of the data in the `dtb.checksum` file, and +cannot be disabled, only replaced with a specific algorithm. + +By default, the hash is `sha256` on 32-bit systems and `sha512` on 64-bit +systems; `sha384` is also available. If a specific algorithm is specified, it +overrides the `hash` default. Older versions of newfol only supported the +`sha256` algorithm. + +The other available hook is `git`. This hook performs a commit every time the +file is saved, and a git checkout of the `dtb` file before every load. This +allows a user to keep a history of the data and to easily back it up to another +location. + +A hook directive consists of a record starting with the value `txn`, and +followed by a list of hook names. `hook` may be used as a synonym for `txn`. + === Configuration File Configuration files are identical to schema files except that instead of the diff --git a/lib/newfol/database.py b/lib/newfol/database.py index ca864d8..3a89dd5 100644 --- a/lib/newfol/database.py +++ b/lib/newfol/database.py @@ -169,7 +169,7 @@ class Schema: self._nfields = nfields self._mapping = {} self._keyfields = [] - self._exe_ok = False + self._exe_ok = None self._imports = [] self._layout = list(range(nfields)) self._cheatsheet = None @@ -185,7 +185,7 @@ class Schema: return self._txntype def execution_allowed(self): - return self._exe_ok + return self._exe_ok or False def mapping(self): return self._mapping @@ -329,7 +329,7 @@ class Schema: mapping = {} keyfields = [] nfields = 0 - exe_ok = False + exe_ok = None exe_seen = False for i in recs: if len(i.fields) == 0: @@ -370,6 +370,8 @@ class Schema: exe_seen = True if i.fields[1].lower() in ("yes", "oui", "sí"): exe_ok = True + else: + exe_ok = False elif rectype in ['imp', 'import']: self._imports = filter(lambda x: x, i.fields[1:]) elif rectype in ['pvu', 'preview']: @@ -381,8 +383,14 @@ class Schema: else: self._cheatsheet = path + "/" + cheatsheet elif rectype in ['col', 'column']: - self._columns = int(i.fields[1]) - elif rectype in ['txn', 'transaction']: + try: + self._columns = int(i.fields[1]) + if self._columns <= 0: + raise ValueError + except ValueError: + raise newfol.exception.SchemaError("has invalid " + + "number of columns") + elif rectype in ['txn', 'transaction', 'hook']: self._txntype = i.fields[1:] elif rectype in ['dpy', 'display']: self._layout = list(map(fix_up_layout, i.fields[1:])) @@ -400,7 +408,10 @@ class Schema: self._keys[i.fields[1]] = None self._mapping = mapping self._keyfields = keyfields - self._exe_ok = exe_ok + if self._exe_ok is None: + self._exe_ok = exe_ok + elif exe_ok is False: + self._exe_ok = False class Database: diff --git a/lib/newfol/exception.py b/lib/newfol/exception.py index 3311076..7e99486 100644 --- a/lib/newfol/exception.py +++ b/lib/newfol/exception.py @@ -15,7 +15,7 @@ class LocaleError(NewfolError): class SchemaError(NewfolError): - def __init__(msg): + def __init__(self, msg): super().__init__("Schema file %s" % msg) @@ -28,7 +28,7 @@ class FilemanipError(Exception): class CorruptFileError(FilemanipError): - def __init__(type, msg): + def __init__(self, type, msg): super().__init__("%s file is corrupt (%s)" % (type, msg)) diff --git a/lib/newfol/filemanip.py b/lib/newfol/filemanip.py index 1403289..0fde28d 100644 --- a/lib/newfol/filemanip.py +++ b/lib/newfol/filemanip.py @@ -61,7 +61,7 @@ class Record: return self._version -class TransactionStore: +class Hook: def prepare_open(self, filename, mode): pass @@ -87,7 +87,7 @@ class TransactionStore: pass -class StackingTransactionStore(TransactionStore): +class StackingHook(Hook): def __init__(self, stores): self._stores = stores @@ -124,7 +124,7 @@ class StackingTransactionStore(TransactionStore): store.commit_store(rec) -class GitTransactionStore(TransactionStore): +class GitHook(Hook): class DirectoryChanger: @@ -198,7 +198,7 @@ class GitTransactionStore(TransactionStore): self._call_git("commit", "-m", message) -class HashTransactionStore(TransactionStore): +class HashHook(Hook): def __init__(self, hashname="sha256", options=None): self._filename = None self._mode = None @@ -230,7 +230,7 @@ class HashTransactionStore(TransactionStore): return choice1 if choice2 is not None: return choice2 - return "sha512" if HashTransactionStore._bitness() == 64 else "sha256" + return "sha512" if HashHook._bitness() == 64 else "sha256" def prepare_open(self, filename, mode): self._filename = filename @@ -578,11 +578,11 @@ class FileStorage: @staticmethod def _get_transaction_types(): return { - "git": GitTransactionStore, - "sha256": lambda *a, **k: HashTransactionStore("sha256", *a, **k), - "sha384": lambda *a, **k: HashTransactionStore("sha384", *a, **k), - "sha512": lambda *a, **k: HashTransactionStore("sha512", *a, **k), - "hash": lambda *a, **k: HashTransactionStore(None, *a, **k), + "git": GitHook, + "sha256": lambda *a, **k: HashHook("sha256", *a, **k), + "sha384": lambda *a, **k: HashHook("sha384", *a, **k), + "sha512": lambda *a, **k: HashHook("sha512", *a, **k), + "hash": lambda *a, **k: HashHook(None, *a, **k), } @staticmethod @@ -593,13 +593,13 @@ class FileStorage: def _make_transaction_store(items, options): txntypes = FileStorage._get_transaction_types() if items is None or items == "": - return TransactionStore() + return Hook() elif isinstance(items, str): return txntypes[items](options) else: stores = [FileStorage._make_transaction_store(i, options) for i in items] - return StackingTransactionStore(stores) + return StackingHook(stores) def store(self, records): """Store the records.""" diff --git a/test/testdatabase.py b/test/testdatabase.py index ea42ad8..b5de4d4 100755 --- a/test/testdatabase.py +++ b/test/testdatabase.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +from newfol.exception import SchemaError from newfol.database import DatabaseVersion, Database, Schema, Singleton from newfol.filemanip import Record import tempfile @@ -59,60 +60,56 @@ class TestDatabaseAccessors(unittest.TestCase): DatabaseVersion.preferred().serialization()) -class TestDatabaseIntegrity(unittest.TestCase): - def create_temp_db(self): - ddir = tempfile.TemporaryDirectory() - with open(ddir.name + "/schema", "w") as fp: - fp.write("fmt:0:newfol schema file:\ntxn:git\n") - db = Database.load(ddir.name) - return (ddir, db) +class TemporaryDatabase: + def __init__(self, schema_contents=""): + self.ddir = tempfile.TemporaryDirectory() + with open(self.ddir.name + "/schema", "w") as fp: + fp.write("fmt:3:newfol schema file:\n" + schema_contents) + @property + def db(self): + return Database.load(self.ddir.name) + + def __del__(self): + self.ddir.cleanup() + + +class TestDatabaseIntegrity(unittest.TestCase): def test_version(self): - ddir, db = self.create_temp_db() - self.assertEqual(Database.read_version(ddir.name), + tdb = TemporaryDatabase() + self.assertEqual(Database.read_version(tdb.ddir.name), DatabaseVersion()) - db.store() - db.upgrade() - self.assertEqual(Database.read_version(ddir.name), + tdb.db.store() + tdb.db.upgrade() + self.assertEqual(Database.read_version(tdb.ddir.name), DatabaseVersion.preferred()) - ddir.cleanup() def test_validate(self): - ddir, db = self.create_temp_db() - db.store() - db.validate() - ddir.cleanup() + tdb = TemporaryDatabase() + tdb.db.store() + tdb.db.validate() def test_validate_strict(self): - ddir, db = self.create_temp_db() - db.store() - db.validate(strict=True) - ddir.cleanup() + tdb = TemporaryDatabase() + tdb.db.store() + tdb.db.validate(strict=True) def test_repair_doesnt_raise(self): - ddir, db = self.create_temp_db() - db.store() - db.repair() - db.validate(strict=True) - ddir.cleanup() + tdb = TemporaryDatabase() + tdb.db.store() + tdb.db.repair() + tdb.db.validate(strict=True) def test_upgrade_records(self): - ddir, db = self.create_temp_db() - db.store() - db.upgrade_records() - ddir.cleanup() + tdb = TemporaryDatabase() + tdb.db.store() + tdb.db.upgrade_records() class TestDatabaseUpgrades(unittest.TestCase): - def create_temp_db(self): - ddir = tempfile.TemporaryDirectory() - with open(ddir.name + "/schema", "w") as fp: - fp.write("fmt:0:newfol schema file:\ntxn:git\n") - db = Database.load(ddir.name) - return (ddir, db) - def do_upgrade_test(self, version, pattern): - ddir, db = self.create_temp_db() + tdb = TemporaryDatabase("txn:git\n") + ddir, db = tdb.ddir, tdb.db if not isinstance(version, DatabaseVersion): version = DatabaseVersion(version) self.assertEqual(Database.read_version(ddir.name), @@ -141,56 +138,84 @@ class TestDatabaseUpgrades(unittest.TestCase): class TestMultipleTransactions(unittest.TestCase): def test_multiple_types(self): - ddir = tempfile.TemporaryDirectory() - with open(ddir.name + "/schema", "w") as fp: - fp.write("fmt:0:newfol schema file:\ntxn:git:hash\n") + tdb = TemporaryDatabase("txn:git:hash\n") + ddir = tdb.ddir db = Database.load(ddir.name) db.records()[:] = [Record([1, 2, 3])] db.store() self.assertEqual(set(db.schema().transaction_types()), set(["git", "hash"])) - ddir.cleanup() class TestExtraSchemaConfig(unittest.TestCase): def test_existing_config_file(self): - ddir1 = tempfile.TemporaryDirectory() + tdb = TemporaryDatabase() ddir2 = tempfile.TemporaryDirectory() config = "%s/config" % ddir2.name - with open(ddir1.name + "/schema", "w") as fp: - fp.write("fmt:3:newfol schema file:\n") with open(config, "w") as fp: fp.write("fmt:3:newfol config file:\ntxn:git:hash\n") - db = Database.load(ddir1.name, extra_config=[config]) + db = Database.load(tdb.ddir.name, extra_config=[config]) db.records()[:] = [Record([1, 2, 3])] db.store() self.assertEqual(set(db.schema().transaction_types()), set(["git", "hash"])) - ddir1.cleanup() ddir2.cleanup() def test_missing_config_file(self): - ddir1 = tempfile.TemporaryDirectory() - config = "%s/config" % ddir1.name - with open(ddir1.name + "/schema", "w") as fp: - fp.write("fmt:3:newfol schema file:\n") - db = Database.load(ddir1.name, extra_config=[config]) + tdb = TemporaryDatabase() + config = "%s/config" % tdb.ddir.name + db = Database.load(tdb.ddir.name, extra_config=[config]) db.records()[:] = [Record([1, 2, 3])] db.store() # Ensure no exception is raised. - ddir1.cleanup() + + +class TestSchemaColumns(unittest.TestCase): + def check_invalid(self, value): + with self.assertRaises(SchemaError): + tdb = TemporaryDatabase("col:%s\n" % value) + tdb.db + + def test_non_integral(self): + self.check_invalid(3.5) + + def test_negative(self): + self.check_invalid(-1) + + def test_zero(self): + self.check_invalid(0) + + +class TestExecutionAllowed(unittest.TestCase): + def do_test(self, expected, schema, configv): + tdb = TemporaryDatabase(schema) + ddir2 = tempfile.TemporaryDirectory() + config = "%s/config" % ddir2.name + with open(config, "w") as fp: + fp.write("fmt:3:newfol config file:\n" + configv) + db = Database.load(tdb.ddir.name, extra_config=[config]) + db.records()[:] = [Record([1, 2, 3])] + db.store() + self.assertEqual(db.schema().execution_allowed(), expected) + ddir2.cleanup() + + def test_true_if_only_true_schema(self): + self.do_test(True, "exe:yes\n", "") + + def test_true_if_only_true_config(self): + self.do_test(True, "", "exe:yes\n") + + def test_false_if_schema_false(self): + self.do_test(False, "exe:no\n", "exe:yes\n") + + def test_false_if_config_false(self): + self.do_test(False, "exe:yes\n", "exe:no\n") class TestDatabaseFiltering(unittest.TestCase): - def create_temp_db(self): - ddir = tempfile.TemporaryDirectory() - with open(ddir.name + "/schema", "w") as fp: - fp.write("fmt:0:newfol schema file:\ntxn:git\n") - db = Database.load(ddir.name) - return (ddir, db) - def test_filtering(self): - ddir, db = self.create_temp_db() + tdb = TemporaryDatabase("txn:git\n") + db = tdb.db records = [ Record(["a", "b", "c"]), Record([1, 2, 3]), @@ -208,7 +233,6 @@ class TestDatabaseFiltering(unittest.TestCase): selected = db.records(has_only_numbers) self.assertEqual(type(selected), list) self.assertEqual(set(selected), set(records[1:])) - ddir.cleanup() class TestSingleton(unittest.TestCase): diff --git a/test/testfilemanip.py b/test/testfilemanip.py index f666b0e..dce504e 100755 --- a/test/testfilemanip.py +++ b/test/testfilemanip.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -from newfol.filemanip import Record, FileStorage, HashTransactionStore +from newfol.filemanip import Record, FileStorage, HashHook import hashlib import newfol.exception import tempfile @@ -140,7 +140,7 @@ class SHA256TransactionTest(unittest.TestCase): v = self.generate_len_and_hash(hashlib.sha256, k) with open(temp, "w") as wfp: wfp.write(k) - self.assertEqual(HashTransactionStore._hash_file("sha256", temp), + self.assertEqual(HashHook._hash_file("sha256", temp), "sha256:" + v) tempdir.cleanup() @@ -210,7 +210,7 @@ class SHA256TransactionTest(unittest.TestCase): x.fs.store([Record(["", ""])]) with open(x.tempfile + ".checksum", "r") as fp: data = fp.read() - bitness = HashTransactionStore._bitness() + bitness = HashHook._bitness() self.assertIn(bitness, [32, 64]) self.assertTrue(data.startswith("sha512" if bitness == 64 else "sha256"))