pestctrl/newfol
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into peter

Browse source
This commit is contained in:
brian m. carlson 2014-12-16 04:00:47 +00:00
commit 13c16acdd8
6 changed files with 199 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
doc/manual.adoc
View file

@ -50,6 +50,85 @@ version on start. newfol does not accept schema files that have a newer version
than it currently supports, but it ignores directives it does not understand,
and so can be manually downgraded by editing the `fmt` directive.
Any record beginning with a `#` is ignored as a comment and preserved on
upgrades. Extra fields in a record are ignored.
The standard rules for formatting colon-separated values apply. In other words,
the format is defined as identical to the `text/csv` format specified in RFC
4180, except that the delimiter is a colon instead of a comma and that the line
break is a single linefeed instead of a CRLF pair.
==== Format Directives (`fmt`)
Format directives define the file as a schema file or configuration file as
specified in <<_schema_file>> and <<_configuration_file>>. They must appear
only at the very beginning of a schema or configuration file. In particular, it
is not allowed for them to be preceded by a byte order mark.
==== Table Definition Directives (`def`)
A table definition directive consists of a record containing the values `def`
and the table name. A table must be defined before any field definitions can
occur for that table.
==== Field Definition Directives (`fld` and `dsc`)
Field definitions come in two types. The first, `fld`, provides a mapping for
imported data, and `dsc` does not. They are otherwise identical.
A field definition directive consists of a record containing the values `fld` or
`dsc`, the name of the table, the field number for imported data (or empty for
`dsc`), the field number for use in newfol, an optional Python 3 expression, and
a description.
The Python 3 expression is evaluated and placed in the field when creating a new
record. This can be used, for example, to automatically insert the date into a
particular field. This requires that code execution has been allowed by an
`exe` directive.
==== Execution Directives (`exe` and `execute`)
Execution directives specify whether Python code will be run from the schema
file. Such a directive consists of a record containing either `exe` or
`execute` and then either `yes` or `no`.
Unlike other directives, where the schema file overrides the configuration file,
if any execution directive is set to `no`, execution is disabled. This is a
security feature to allow reading an untrusted database. Execution is also
disabled if more than one execution directive is seen in either the schema or
configuration file.
==== Key Field Directives (`key`)
Key field directives indicate the fields to display in the _Browse All Records_
view. By default, this value is 0 (display the first field).
==== Column Directives (`col`)
Column directives indicate how many columns to use to display fields. A column
directive consists of a record with the values `col` and an integer number of
columns.
==== Hook Directives (`txn` and `hook`)
Hook directives indicate a series of hooks to be run on each load or save
operation. Several different hooks are available. The default set of hooks is
`hash`. This provides an SHA-2 hash of the data in the `dtb.checksum` file, and
cannot be disabled, only replaced with a specific algorithm.
By default, the hash is `sha256` on 32-bit systems and `sha512` on 64-bit
systems; `sha384` is also available. If a specific algorithm is specified, it
overrides the `hash` default. Older versions of newfol only supported the
`sha256` algorithm.
The other available hook is `git`. This hook performs a commit every time the
file is saved, and a git checkout of the `dtb` file before every load. This
allows a user to keep a history of the data and to easily back it up to another
location.
A hook directive consists of a record starting with the value `txn`, and
followed by a list of hook names. `hook` may be used as a synonym for `txn`.
=== Configuration File
Configuration files are identical to schema files except that instead of the

23
lib/newfol/database.py
View file

@ -169,7 +169,7 @@ class Schema:
self._nfields = nfields
self._mapping = {}
self._keyfields = []
self._exe_ok = False
self._exe_ok = None
self._imports = []
self._layout = list(range(nfields))
self._cheatsheet = None
@ -185,7 +185,7 @@ class Schema:
return self._txntype
def execution_allowed(self):
return self._exe_ok
return self._exe_ok or False
def mapping(self):
return self._mapping
@ -329,7 +329,7 @@ class Schema:
mapping = {}
keyfields = []
nfields = 0
exe_ok = False
exe_ok = None
exe_seen = False
for i in recs:
if len(i.fields) == 0:
@ -370,6 +370,8 @@ class Schema:
exe_seen = True
if i.fields[1].lower() in ("yes", "oui", "sí"):
exe_ok = True
else:
exe_ok = False
elif rectype in ['imp', 'import']:
self._imports = filter(lambda x: x, i.fields[1:])
elif rectype in ['pvu', 'preview']:
@ -381,8 +383,14 @@ class Schema:
else:
self._cheatsheet = path + "/" + cheatsheet
elif rectype in ['col', 'column']:
self._columns = int(i.fields[1])
elif rectype in ['txn', 'transaction']:
try:
self._columns = int(i.fields[1])
if self._columns <= 0:
raise ValueError
except ValueError:
raise newfol.exception.SchemaError("has invalid " +
"number of columns")
elif rectype in ['txn', 'transaction', 'hook']:
self._txntype = i.fields[1:]
elif rectype in ['dpy', 'display']:
self._layout = list(map(fix_up_layout, i.fields[1:]))
@ -400,7 +408,10 @@ class Schema:
self._keys[i.fields[1]] = None
self._mapping = mapping
self._keyfields = keyfields
self._exe_ok = exe_ok
if self._exe_ok is None:
self._exe_ok = exe_ok
elif exe_ok is False:
self._exe_ok = False
class Database:

4
lib/newfol/exception.py
View file

@ -15,7 +15,7 @@ class LocaleError(NewfolError):
class SchemaError(NewfolError):
def __init__(msg):
def __init__(self, msg):
super().__init__("Schema file %s" % msg)
@ -28,7 +28,7 @@ class FilemanipError(Exception):
class CorruptFileError(FilemanipError):
def __init__(type, msg):
def __init__(self, type, msg):
super().__init__("%s file is corrupt (%s)" % (type, msg))

24
lib/newfol/filemanip.py
View file

@ -61,7 +61,7 @@ class Record:
return self._version
class TransactionStore:
class Hook:
def prepare_open(self, filename, mode):
pass
@ -87,7 +87,7 @@ class TransactionStore:
pass
class StackingTransactionStore(TransactionStore):
class StackingHook(Hook):
def __init__(self, stores):
self._stores = stores
@ -124,7 +124,7 @@ class StackingTransactionStore(TransactionStore):
store.commit_store(rec)
class GitTransactionStore(TransactionStore):
class GitHook(Hook):
class DirectoryChanger:
@ -198,7 +198,7 @@ class GitTransactionStore(TransactionStore):
self._call_git("commit", "-m", message)
class HashTransactionStore(TransactionStore):
class HashHook(Hook):
def __init__(self, hashname="sha256", options=None):
self._filename = None
self._mode = None
@ -230,7 +230,7 @@ class HashTransactionStore(TransactionStore):
return choice1
if choice2 is not None:
return choice2
return "sha512" if HashTransactionStore._bitness() == 64 else "sha256"
return "sha512" if HashHook._bitness() == 64 else "sha256"
def prepare_open(self, filename, mode):
self._filename = filename
@ -578,11 +578,11 @@ class FileStorage:
@staticmethod
def _get_transaction_types():
return {
"git": GitTransactionStore,
"sha256": lambda *a, **k: HashTransactionStore("sha256", *a, **k),
"sha384": lambda *a, **k: HashTransactionStore("sha384", *a, **k),
"sha512": lambda *a, **k: HashTransactionStore("sha512", *a, **k),
"hash": lambda *a, **k: HashTransactionStore(None, *a, **k),
"git": GitHook,
"sha256": lambda *a, **k: HashHook("sha256", *a, **k),
"sha384": lambda *a, **k: HashHook("sha384", *a, **k),
"sha512": lambda *a, **k: HashHook("sha512", *a, **k),
"hash": lambda *a, **k: HashHook(None, *a, **k),
}
@staticmethod
@ -593,13 +593,13 @@ class FileStorage:
def _make_transaction_store(items, options):
txntypes = FileStorage._get_transaction_types()
if items is None or items == "":
return TransactionStore()
return Hook()
elif isinstance(items, str):
return txntypes[items](options)
else:
stores = [FileStorage._make_transaction_store(i, options)
for i in items]
return StackingTransactionStore(stores)
return StackingHook(stores)
def store(self, records):
"""Store the records."""

148
test/testdatabase.py
View file

@ -1,5 +1,6 @@
#!/usr/bin/python3
from newfol.exception import SchemaError
from newfol.database import DatabaseVersion, Database, Schema, Singleton
from newfol.filemanip import Record
import tempfile
@ -59,60 +60,56 @@ class TestDatabaseAccessors(unittest.TestCase):
DatabaseVersion.preferred().serialization())
class TestDatabaseIntegrity(unittest.TestCase):
def create_temp_db(self):
ddir = tempfile.TemporaryDirectory()
with open(ddir.name + "/schema", "w") as fp:
fp.write("fmt:0:newfol schema file:\ntxn:git\n")
db = Database.load(ddir.name)
return (ddir, db)
class TemporaryDatabase:
def __init__(self, schema_contents=""):
self.ddir = tempfile.TemporaryDirectory()
with open(self.ddir.name + "/schema", "w") as fp:
fp.write("fmt:3:newfol schema file:\n" + schema_contents)
@property
def db(self):
return Database.load(self.ddir.name)
def __del__(self):
self.ddir.cleanup()
class TestDatabaseIntegrity(unittest.TestCase):
def test_version(self):
ddir, db = self.create_temp_db()
self.assertEqual(Database.read_version(ddir.name),
tdb = TemporaryDatabase()
self.assertEqual(Database.read_version(tdb.ddir.name),
DatabaseVersion())
db.store()
db.upgrade()
self.assertEqual(Database.read_version(ddir.name),
tdb.db.store()
tdb.db.upgrade()
self.assertEqual(Database.read_version(tdb.ddir.name),
DatabaseVersion.preferred())
ddir.cleanup()
def test_validate(self):
ddir, db = self.create_temp_db()
db.store()
db.validate()
ddir.cleanup()
tdb = TemporaryDatabase()
tdb.db.store()
tdb.db.validate()
def test_validate_strict(self):
ddir, db = self.create_temp_db()
db.store()
db.validate(strict=True)
ddir.cleanup()
tdb = TemporaryDatabase()
tdb.db.store()
tdb.db.validate(strict=True)
def test_repair_doesnt_raise(self):
ddir, db = self.create_temp_db()
db.store()
db.repair()
db.validate(strict=True)
ddir.cleanup()
tdb = TemporaryDatabase()
tdb.db.store()
tdb.db.repair()
tdb.db.validate(strict=True)
def test_upgrade_records(self):
ddir, db = self.create_temp_db()
db.store()
db.upgrade_records()
ddir.cleanup()
tdb = TemporaryDatabase()
tdb.db.store()
tdb.db.upgrade_records()
class TestDatabaseUpgrades(unittest.TestCase):
def create_temp_db(self):
ddir = tempfile.TemporaryDirectory()
with open(ddir.name + "/schema", "w") as fp:
fp.write("fmt:0:newfol schema file:\ntxn:git\n")
db = Database.load(ddir.name)
return (ddir, db)
def do_upgrade_test(self, version, pattern):
ddir, db = self.create_temp_db()
tdb = TemporaryDatabase("txn:git\n")
ddir, db = tdb.ddir, tdb.db
if not isinstance(version, DatabaseVersion):
version = DatabaseVersion(version)
self.assertEqual(Database.read_version(ddir.name),
@ -141,56 +138,84 @@ class TestDatabaseUpgrades(unittest.TestCase):
class TestMultipleTransactions(unittest.TestCase):
def test_multiple_types(self):
ddir = tempfile.TemporaryDirectory()
with open(ddir.name + "/schema", "w") as fp:
fp.write("fmt:0:newfol schema file:\ntxn:git:hash\n")
tdb = TemporaryDatabase("txn:git:hash\n")
ddir = tdb.ddir
db = Database.load(ddir.name)
db.records()[:] = [Record([1, 2, 3])]
db.store()
self.assertEqual(set(db.schema().transaction_types()),
set(["git", "hash"]))
ddir.cleanup()
class TestExtraSchemaConfig(unittest.TestCase):
def test_existing_config_file(self):
ddir1 = tempfile.TemporaryDirectory()
tdb = TemporaryDatabase()
ddir2 = tempfile.TemporaryDirectory()
config = "%s/config" % ddir2.name
with open(ddir1.name + "/schema", "w") as fp:
fp.write("fmt:3:newfol schema file:\n")
with open(config, "w") as fp:
fp.write("fmt:3:newfol config file:\ntxn:git:hash\n")
db = Database.load(ddir1.name, extra_config=[config])
db = Database.load(tdb.ddir.name, extra_config=[config])
db.records()[:] = [Record([1, 2, 3])]
db.store()
self.assertEqual(set(db.schema().transaction_types()),
set(["git", "hash"]))
ddir1.cleanup()
ddir2.cleanup()
def test_missing_config_file(self):
ddir1 = tempfile.TemporaryDirectory()
config = "%s/config" % ddir1.name
with open(ddir1.name + "/schema", "w") as fp:
fp.write("fmt:3:newfol schema file:\n")
db = Database.load(ddir1.name, extra_config=[config])
tdb = TemporaryDatabase()
config = "%s/config" % tdb.ddir.name
db = Database.load(tdb.ddir.name, extra_config=[config])
db.records()[:] = [Record([1, 2, 3])]
db.store()
# Ensure no exception is raised.
ddir1.cleanup()
class TestSchemaColumns(unittest.TestCase):
def check_invalid(self, value):
with self.assertRaises(SchemaError):
tdb = TemporaryDatabase("col:%s\n" % value)
tdb.db
def test_non_integral(self):
self.check_invalid(3.5)
def test_negative(self):
self.check_invalid(-1)
def test_zero(self):
self.check_invalid(0)
class TestExecutionAllowed(unittest.TestCase):
def do_test(self, expected, schema, configv):
tdb = TemporaryDatabase(schema)
ddir2 = tempfile.TemporaryDirectory()
config = "%s/config" % ddir2.name
with open(config, "w") as fp:
fp.write("fmt:3:newfol config file:\n" + configv)
db = Database.load(tdb.ddir.name, extra_config=[config])
db.records()[:] = [Record([1, 2, 3])]
db.store()
self.assertEqual(db.schema().execution_allowed(), expected)
ddir2.cleanup()
def test_true_if_only_true_schema(self):
self.do_test(True, "exe:yes\n", "")
def test_true_if_only_true_config(self):
self.do_test(True, "", "exe:yes\n")
def test_false_if_schema_false(self):
self.do_test(False, "exe:no\n", "exe:yes\n")
def test_false_if_config_false(self):
self.do_test(False, "exe:yes\n", "exe:no\n")
class TestDatabaseFiltering(unittest.TestCase):
def create_temp_db(self):
ddir = tempfile.TemporaryDirectory()
with open(ddir.name + "/schema", "w") as fp:
fp.write("fmt:0:newfol schema file:\ntxn:git\n")
db = Database.load(ddir.name)
return (ddir, db)
def test_filtering(self):
ddir, db = self.create_temp_db()
tdb = TemporaryDatabase("txn:git\n")
db = tdb.db
records = [
Record(["a", "b", "c"]),
Record([1, 2, 3]),
@ -208,7 +233,6 @@ class TestDatabaseFiltering(unittest.TestCase):
selected = db.records(has_only_numbers)
self.assertEqual(type(selected), list)
self.assertEqual(set(selected), set(records[1:]))
ddir.cleanup()
class TestSingleton(unittest.TestCase):

6
test/testfilemanip.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/python3
from newfol.filemanip import Record, FileStorage, HashTransactionStore
from newfol.filemanip import Record, FileStorage, HashHook
import hashlib
import newfol.exception
import tempfile
@ -140,7 +140,7 @@ class SHA256TransactionTest(unittest.TestCase):
v = self.generate_len_and_hash(hashlib.sha256, k)
with open(temp, "w") as wfp:
wfp.write(k)
self.assertEqual(HashTransactionStore._hash_file("sha256", temp),
self.assertEqual(HashHook._hash_file("sha256", temp),
"sha256:" + v)
tempdir.cleanup()
@ -210,7 +210,7 @@ class SHA256TransactionTest(unittest.TestCase):
x.fs.store([Record(["", ""])])
with open(x.tempfile + ".checksum", "r") as fp:
data = fp.read()
bitness = HashTransactionStore._bitness()
bitness = HashHook._bitness()
self.assertIn(bitness, [32, 64])
self.assertTrue(data.startswith("sha512" if bitness == 64 else
"sha256"))