Github/emacs
2
0
Fork 1
mirror of git://git.sv.gnu.org/emacs.git synced 2026-02-19 19:37:58 +00:00
Code Issues Projects Releases Packages Wiki Activity

Catch-up merge from master sources to mps/branch/2013-05-17/emergency.

Browse source 
Copied from Perforce
 Change: 182518
 ServerID: perforce.ravenbrook.com
This commit is contained in:
Gareth Rees 2013-06-04 20:44:03 +01:00
commit 0b204bf4e7
316 changed files with 31464 additions and 4586 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
mps/code/commpost.nmk
View file

@ -1,44 +1,21 @@
# commpost.nmk: SECOND COMMON FRAGMENT FOR PLATFORMS USING MV AND NMAKE
# commpost.nmk: SECOND COMMON FRAGMENT FOR PLATFORMS USING NMAKE -*- makefile -*-
#
# $Id$
# Copyright (c) 2001 Ravenbrook Limited. See end of file for license.
# Copyright (c) 2001-2013 Ravenbrook Limited. See end of file for license.
#
# DESCRIPTION
#
# Second common makefile fragment for w3*mv.nmk. See commpre.nmk
# PSEUDO-TARGETS
# == Pseudo-targets ==
# "all" builds all the varieties of all targets
# %%TARGET: When adding a new target, add it to the all dependencies.
all: mpmss.exe amcss.exe amsss.exe amssshe.exe segsmss.exe awlut.exe awluthe.exe\
mpsicv.exe lockutw3.exe lockcov.exe poolncv.exe locv.exe qs.exe apss.exe \
sacss.exe finalcv.exe finaltest.exe \
arenacv.exe bttest.exe teletest.exe \
abqtest.exe cbstest.exe btcv.exe mv2test.exe messtest.exe steptest.exe \
locbwcss.exe locusss.exe zcoll.exe zmess.exe \
mpseventcnv.exe mpseventtxt.exe \
mps.lib
all: $(ALL_TARGETS)
# Convenience targets
# %%TARGET: When adding a new target, add a pseudo-target for it here,
# first rule for variety-dependent targets, and second for
# variety-independent ones.
mpmss.exe amcss.exe amcsshe.exe amsss.exe amssshe.exe segsmss.exe awlut.exe awluthe.exe dwstress.exe \
mpsicv.exe lockutw3.exe lockcov.exe poolncv.exe locv.exe qs.exe apss.exe \
sacss.exe finalcv.exe finaltest.exe \
arenacv.exe bttest.exe teletest.exe \
expt825.exe \
abqtest.exe cbstest.exe btcv.exe mv2test.exe messtest.exe steptest.exe \
walkt0.exe locbwcss.exe locusss.exe \
exposet0.exe zcoll.exe zmess.exe \
replay.exe replaysw.exe mpseventcnv.exe mpseventtxt.exe mpseventsql.exe \
mps.lib:
$(ALL_TARGETS):
!IFDEF VARIETY
$(MAKE) /nologo /f $(PFM).nmk TARGET=$@ variety
!ELSE
@ -79,6 +56,17 @@ variety: $(PFM)\$(VARIETY)\$(TARGET)
mpsicv.cov:
$(MAKE) /nologo /f $(PFM).nmk TARGET=$@ VARIETY=cv variety
# testrun
# Runs automated test cases.
testrun: $(AUTO_TEST_TARGETS)
!IFDEF VARIETY
..\tool\testrun.bat $(PFM) $(VARIETY) $(AUTO_TEST_TARGETS)
!ELSE
$(MAKE) /nologo /f $(PFM).nmk VARIETY=hot testrun
$(MAKE) /nologo /f $(PFM).nmk VARIETY=cool testrun
!ENDIF
# THE MPS LIBRARY
#
@ -133,41 +121,8 @@ $(PFM)\di\mps.lib: \
!IFDEF VARIETY
$(PFM)\$(VARIETY)\finalcv.exe: $(PFM)\$(VARIETY)\finalcv.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\finaltest.exe: $(PFM)\$(VARIETY)\finaltest.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\expt825.exe: $(PFM)\$(VARIETY)\expt825.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\locv.exe: $(PFM)\$(VARIETY)\locv.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\mpmss.exe: $(PFM)\$(VARIETY)\mpmss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\apss.exe: $(PFM)\$(VARIETY)\apss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\sacss.exe: $(PFM)\$(VARIETY)\sacss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\bttest.exe: $(PFM)\$(VARIETY)\bttest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\teletest.exe: $(PFM)\$(VARIETY)\teletest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\lockcov.exe: $(PFM)\$(VARIETY)\lockcov.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\lockutw3.exe: $(PFM)\$(VARIETY)\lockutw3.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\mpsicv.exe: $(PFM)\$(VARIETY)\mpsicv.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\abqtest.exe: $(PFM)\$(VARIETY)\abqtest.obj \
$(PFM)\$(VARIETY)\abq.obj $(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\amcss.exe: $(PFM)\$(VARIETY)\amcss.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
@ -175,24 +130,21 @@ $(PFM)\$(VARIETY)\amcss.exe: $(PFM)\$(VARIETY)\amcss.obj \
$(PFM)\$(VARIETY)\amcsshe.exe: $(PFM)\$(VARIETY)\amcsshe.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\amcssth.exe: $(PFM)\$(VARIETY)\amcssth.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\amsss.exe: $(PFM)\$(VARIETY)\amsss.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\amssshe.exe: $(PFM)\$(VARIETY)\amssshe.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\segsmss.exe: $(PFM)\$(VARIETY)\segsmss.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\locbwcss.exe: $(PFM)\$(VARIETY)\locbwcss.obj \
$(PFM)\$(VARIETY)\apss.exe: $(PFM)\$(VARIETY)\apss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\locusss.exe: $(PFM)\$(VARIETY)\locusss.obj \
$(PFM)\$(VARIETY)\arenacv.exe: $(PFM)\$(VARIETY)\arenacv.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\dwstress.exe: $(PFM)\$(VARIETY)\dwstress.obj \
$(DWOBJ) $(PFM)\$(VARIETY)\mps.lib
$(PFM)\$(VARIETY)\awlut.exe: $(PFM)\$(VARIETY)\awlut.obj \
$(FMTTESTOBJ) \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
@ -201,27 +153,75 @@ $(PFM)\$(VARIETY)\awluthe.exe: $(PFM)\$(VARIETY)\awluthe.obj \
$(FMTTESTOBJ) \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\btcv.exe: $(PFM)\$(VARIETY)\btcv.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\bttest.exe: $(PFM)\$(VARIETY)\bttest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\cbstest.exe: $(PFM)\$(VARIETY)\cbstest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\exposet0.exe: $(PFM)\$(VARIETY)\exposet0.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\expt825.exe: $(PFM)\$(VARIETY)\expt825.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\finalcv.exe: $(PFM)\$(VARIETY)\finalcv.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\finaltest.exe: $(PFM)\$(VARIETY)\finaltest.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\locbwcss.exe: $(PFM)\$(VARIETY)\locbwcss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\lockcov.exe: $(PFM)\$(VARIETY)\lockcov.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\lockutw3.exe: $(PFM)\$(VARIETY)\lockutw3.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\locusss.exe: $(PFM)\$(VARIETY)\locusss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\locv.exe: $(PFM)\$(VARIETY)\locv.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\messtest.exe: $(PFM)\$(VARIETY)\messtest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\mpmss.exe: $(PFM)\$(VARIETY)\mpmss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\mpsicv.exe: $(PFM)\$(VARIETY)\mpsicv.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\mv2test.exe: $(PFM)\$(VARIETY)\mv2test.obj \
$(PFM)\$(VARIETY)\poolmv2.obj $(PFM)\$(VARIETY)\abq.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\poolncv.exe: $(PFM)\$(VARIETY)\poolncv.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\qs.exe: $(PFM)\$(VARIETY)\qs.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\arenacv.exe: $(PFM)\$(VARIETY)\arenacv.obj \
$(PFM)\$(VARIETY)\sacss.exe: $(PFM)\$(VARIETY)\sacss.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\abqtest.exe: $(PFM)\$(VARIETY)\abqtest.obj \
$(PFM)\$(VARIETY)\abq.obj $(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\segsmss.exe: $(PFM)\$(VARIETY)\segsmss.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\cbstest.exe: $(PFM)\$(VARIETY)\cbstest.obj \
$(PFM)\$(VARIETY)\steptest.exe: $(PFM)\$(VARIETY)\steptest.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\teletest.exe: $(PFM)\$(VARIETY)\teletest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\btcv.exe: $(PFM)\$(VARIETY)\btcv.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\mv2test.exe: $(PFM)\$(VARIETY)\mv2test.obj \
$(PFM)\$(VARIETY)\poolmv2.obj $(PFM)\$(VARIETY)\abq.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\walkt0.exe: $(PFM)\$(VARIETY)\walkt0.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\zcoll.exe: $(PFM)\$(VARIETY)\zcoll.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) \
@ -251,18 +251,6 @@ $(PFM)\$(VARIETY)\replaysw.obj: $(PFM)\$(VARIETY)\replay.obj
$(ECHO) $@
copy $** $@ >nul:
$(PFM)\$(VARIETY)\messtest.exe: $(PFM)\$(VARIETY)\messtest.obj \
$(PFM)\$(VARIETY)\mps.lib $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\steptest.exe: $(PFM)\$(VARIETY)\steptest.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\walkt0.exe: $(PFM)\$(VARIETY)\walkt0.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
$(PFM)\$(VARIETY)\exposet0.exe: $(PFM)\$(VARIETY)\exposet0.obj \
$(PFM)\$(VARIETY)\mps.lib $(FMTTESTOBJ) $(TESTLIBOBJ)
!ENDIF
@ -307,7 +295,7 @@ $(PFM)\$(VARIETY)\sqlite3.obj:
# C. COPYRIGHT AND LICENSE
#
# Copyright (C) 2001-2002 Ravenbrook Limited <http://www.ravenbrook.com/>.
# Copyright (C) 2001-2013 Ravenbrook Limited <http://www.ravenbrook.com/>.
# All rights reserved. This is an open source license. Contact
# Ravenbrook for commercial licensing options.
#

91
mps/code/commpre.nmk
View file

@ -1,4 +1,4 @@
# commpre.nmk: FIRST COMMON FRAGMENT FOR PLATFORMS USING MV AND NMAKE
# commpre.nmk: FIRST COMMON FRAGMENT FOR PLATFORMS USING NMAKE -*- makefile -*-1
#
# $Id$
# Copyright (c) 2001 Ravenbrook Limited. See end of file for license.
@ -14,17 +14,22 @@
# %%PART: When adding a new part, add a new parameter for the files included
# in the part
# Parameters:
# PFM platform code, e.g. "nti3mv"
# PFM platform code, e.g. "w3i3mv"
# PFMDEFS /D options to define platforms preprocessor symbols
# to the compiler. Eg "/DOS_NT /DARCH_386 /DBUILD_MVC"
# MPM list of sources which make up the "mpm" part for this
# platform. Each source is stripped of its .c extension
# MPMCOMMON list of sources which make up the "mpm" part for all
# platforms. Each source is stripped of its .c extension
# and surrounded in angle brackets (<>)
# MPM as above, plus sources for the "mpm" part for the current
# platform.
# PLINTH as above for the "plinth" part
# AMC as above for the "amc" part
# AMS as above for the "ams" part
# LO as above for the "lo" part
# MRG as above for the "mrg" part
# POOLN as above for the "pooln" part
# SNC as above for the "snc" part
# DW as above for the "dw" part
# FMTTEST as above for the "fmttest" part
# TESTLIB as above for the "testlib" part
# NOISY if defined, causes command to be emitted
#
@ -33,11 +38,75 @@
#
# To add new targets. varieties, and parts:
# Search for the string "%%TARGET", "%%VARIETY", or "%%PART" in this makefile
# and follow the instructions. If you're adding a part, you'll have to change
# the makefile for all the platforms which use this makefile to define the
# source list for that part.
# and follow the instructions.
#
# TARGETS
#
#
# %%TARGET: When adding a new target, add it to one of the variables
# in this section. Library components go in LIB_TARGETS.
LIB_TARGETS=mps.lib
# If it is suitable for running regularly (for example, after every
# build) as an automated test case, add it to AUTO_TEST_TARGETS.
AUTO_TEST_TARGETS=abqtest.exe amcss.exe amcsshe.exe amsss.exe \
amssshe.exe apss.exe arenacv.exe awlut.exe awluthe.exe btcv.exe \
cbstest.exe exposet0.exe expt825.exe finalcv.exe finaltest.exe \
locbwcss.exe lockcov.exe lockutw3.exe locusss.exe locv.exe \
messtest.exe mpmss.exe mpsicv.exe mv2test.exe poolncv.exe qs.exe \
sacss.exe segsmss.exe steptest.exe walkt0.exe zmess.exe
# If it is not runnable as an automated test case, but is buildable,
# add it to OTHER_TEST_TARGETS with a note.
#
# bttest and teletest -- interactive and so cannot be run unattended.
# zcoll -- takes too long to be useful as a regularly run smoke test.
OTHER_TEST_TARGETS=bttest.exe teletest.exe zcoll.exe
# Stand-alone programs go in EXTRA_TARGETS.
EXTRA_TARGETS=mpseventcnv.exe mpseventtxt.exe
# This target records programs that we were once able to build but
# can't at the moment:
#
# replay -- depends on the EPVM pool.
UNBUILDABLE_TARGETS=replay.exe
ALL_TARGETS=$(LIB_TARGETS) $(AUTO_TEST_TARGETS) $(OTHER_TEST_TARGETS) $(EXTRA_TARGETS)
# PARAMETERS
#
#
# %%PART: When adding a new part, add the sources for the new part here.
MPMCOMMON = <abq> <arena> <arenacl> <arenavm> <arg> <boot> <bt> \
<buffer> <cbs> <dbgpool> <dbgpooli> <diag> <event> <format> \
<global> <ld> <lockw3> <locus> <message> <meter> <mpm> <mpsi> \
<mpsiw3> <pool> <poolabs> <poolmfs> <poolmrg> <poolmv2> <poolmv> \
<protocol> <protw3> <ref> <reserv> <ring> <root> <sac> <seg> \
<shield> <splay> <ss> <table> <thw3> <trace> <traceanc> <tract> \
<vmw3> <walk>
PLINTH = <mpsliban> <mpsioan>
AMC = <poolamc>
AMS = <poolams> <poolamsi>
AWL = <poolawl>
LO = <poollo>
MVFF = <poolmvff>
POOLN = <pooln>
SNC = <poolsnc>
DW = <fmtdy> <fmtno>
FMTTEST = <fmthe> <fmtdy> <fmtno> <fmtdytst>
TESTLIB = <testlib>
# CHECK PARAMETERS
#
#
@ -50,8 +119,8 @@
!IFNDEF PFMDEFS
!ERROR commpre.nmk: PFMDEFS not defined
!ENDIF
!IFNDEF MPM
!ERROR commpre.nmk: MPM not defined
!IFNDEF MPMCOMMON
!ERROR commpre.nmk: MPMCOMMON not defined
!ENDIF
!IFNDEF PLINTH
!ERROR commpre.nmk: PLINTH not defined
@ -166,7 +235,7 @@ LIBFLAGSCOOL =
# C. COPYRIGHT AND LICENSE
#
# Copyright (C) 2001-2002 Ravenbrook Limited <http://www.ravenbrook.com/>.
# Copyright (C) 2001-2013 Ravenbrook Limited <http://www.ravenbrook.com/>.
# All rights reserved. This is an open source license. Contact
# Ravenbrook for commercial licensing options.
#

4
mps/code/mpm.c
View file

@ -493,7 +493,9 @@ Res WriteF_firstformat_v(mps_lib_FILE *stream,
case 'F': { /* function */
WriteFF f = va_arg(args, WriteFF);
Byte *b = (Byte *)&f;
/* TODO: Why do we always write these little-endian? */
/* ISO C forbits casting function pointers to integer, so
decode bytes (see design.writef.f).
TODO: Be smarter about endianness. */
for(i=0; i < sizeof(WriteFF); i++) {
res = WriteULongest(stream, (ULongest)(b[i]), 16,
(CHAR_BIT + 3) / 4);

33
mps/code/w3i3mv.nmk
View file

@ -1,7 +1,7 @@
# w3i3mv.nmk: WINDOWS (INTEL) NMAKE FILE
# w3i3mv.nmk: WINDOWS (IA-32) NMAKE FILE -*- makefile -*-
#
# $Id$
# Copyright (c) 2001 Ravenbrook Limited. See end of file for license.
# Copyright (c) 2001-2013 Ravenbrook Limited. See end of file for license.
PFM = w3i3mv
@ -11,31 +11,12 @@ PFM = w3i3mv
PFMDEFS = /DCONFIG_PF_STRING="w3i3mv" /DCONFIG_PF_W3I3MV \
/DWIN32 /D_WINDOWS /Gs
MPM = <ring> <mpm> <bt> <protocol> <boot> \
<arenavm> <arenacl> <locus> <arena> <global> <tract> <reserv> \
<pool> <poolabs> <poolmfs> <poolmv> \
<root> <format> <buffer> <walk> <lockw3> \
<ref> <trace> <traceanc> <protw3> <proti3> <prmci3w3> \
<shield> <vmw3> <table> \
<thw3> <thw3i3> <ss> <ssw3i3mv> <mpsi> <mpsiw3> <ld> <spi3> \
<event> <seg> <sac> <poolmrg> <message> <dbgpool> <dbgpooli> \
<abq> <meter> <cbs> <poolmv2> <splay> <diag> <arg>
PLINTH = <mpsliban> <mpsioan>
AMC = <poolamc>
AMS = <poolams> <poolamsi>
AWL = <poolawl>
LO = <poollo>
SNC = <poolsnc>
MVFF = <poolmvff>
N = <pooln>
DW = <fmtdy> <fmtno>
FMTTEST = <fmthe> <fmtdy> <fmtno> <fmtdytst>
POOLN = <pooln>
TESTLIB = <testlib>
!INCLUDE commpre.nmk
# MPM sources: core plus platform-specific.
MPM = $(MPMCOMMON) <proti3> <prmci3w3> <spi3> <ssw3i3mv> <thw3i3>
# Source to object file mappings and CFLAGS amalgamation
#
@ -187,7 +168,7 @@ TESTLIBOBJ = $(TESTLIBOBJ0:>=.obj)
# C. COPYRIGHT AND LICENSE
#
# Copyright (C) 2001-2002 Ravenbrook Limited <http://www.ravenbrook.com/>.
# Copyright (C) 2001-2013 Ravenbrook Limited <http://www.ravenbrook.com/>.
# All rights reserved. This is an open source license. Contact
# Ravenbrook for commercial licensing options.
#

28
mps/code/w3i6mv.nmk
View file

@ -1,7 +1,7 @@
# w3i6mv.nmk: WINDOWS (x64) NMAKE FILE
# w3i6mv.nmk: WINDOWS (x86-64) NMAKE FILE -*- makefile -*-
#
# $Id$
# Copyright (c) 2001 Ravenbrook Limited. See end of file for license.
# Copyright (c) 2001-2013 Ravenbrook Limited. See end of file for license.
PFM = w3i6mv
@ -12,26 +12,8 @@ PFMDEFS = /DCONFIG_PF_STRING="w3i6mv" /DCONFIG_PF_W3I6MV \
/DWIN32 /D_WINDOWS /Gs
MASM = ml64
MPM = <ring> <mpm> <bt> <protocol> <boot> \
<arenavm> <arenacl> <locus> <arena> <global> <tract> <reserv> \
<pool> <poolabs> <poolmfs> <poolmv> \
<root> <format> <buffer> <walk> <lockw3> \
<ref> <trace> <traceanc> <protw3> <proti6> <prmci6w3> \
<shield> <vmw3> <table> \
<thw3> <thw3i6> <ss> <ssw3i6mv> <mpsi> <mpsiw3> <ld> <span> \
<event> <seg> <sac> <poolmrg> <message> <dbgpool> <dbgpooli> \
<abq> <meter> <cbs> <poolmv2> <splay> <diag> <arg>
PLINTH = <mpsliban> <mpsioan>
AMC = <poolamc>
AMS = <poolams> <poolamsi>
AWL = <poolawl>
LO = <poollo>
SNC = <poolsnc>
MVFF = <poolmvff>
DW = <fmtdy> <fmtno>
FMTTEST = <fmthe> <fmtdy> <fmtno> <fmtdytst>
POOLN = <pooln>
TESTLIB = <testlib>
# MPM sources: core plus platform-specific.
MPM = $(MPMCOMMON) <prmci6w3> <span> <ssw3i6mv> <thw3i6>
!INCLUDE commpre.nmk
@ -187,7 +169,7 @@ TESTLIBOBJ = $(TESTLIBOBJ0:>=.obj)
# C. COPYRIGHT AND LICENSE
#
# Copyright (C) 2001-2002 Ravenbrook Limited <http://www.ravenbrook.com/>.
# Copyright (C) 2001-2013 Ravenbrook Limited <http://www.ravenbrook.com/>.
# All rights reserved. This is an open source license. Contact
# Ravenbrook for commercial licensing options.
#

4
mps/design/alloc-frame.txt
View file

@ -9,6 +9,7 @@ Allocation frame protocol
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: allocation frames; design
Introduction
@ -252,8 +253,7 @@ the ``SelectFrameOfAddr()`` operation.
_`.fn.client.in-frame`: This function is used by clients to invoke the
``AddrInFrame()`` operation.
``mps_res_t mps_ap_set_frame_class(mps_ap_t buf, mps_frame_class_t
class)``
``mps_res_t mps_ap_set_frame_class(mps_ap_t buf, mps_frame_class_t class)``
_`.fn.client.set`: This function is used by clients to invoke the
``SetFrameClass()`` operation.

1
mps/design/arena.txt
View file

@ -9,6 +9,7 @@ Arena
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: arena; design
Introduction

4
mps/design/arenavm.txt
View file

@ -9,7 +9,9 @@ Virtual Memory Arena
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: virtual memory arena; design
pair: VM arena; design
Introduction

1
mps/design/bt.txt
View file

@ -9,6 +9,7 @@ Bit tables
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: bit tables; design
Introduction

1
mps/design/buffer.txt
View file

@ -9,6 +9,7 @@ Allocation buffers and allocation points
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: buffers; design
Introduction

1
mps/design/check.txt
View file

@ -9,6 +9,7 @@ Checking
:Status: incomplete design
:Revision: $Id$
:Copyright: See section `Copyright and License`_.
:Index terms: pair: checking; design
Introduction

1
mps/design/class-interface.txt
View file

@ -9,6 +9,7 @@ Pool class interface
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: class interface; design
Introduction

1
mps/design/collection.txt
View file

@ -9,6 +9,7 @@ Collection framework
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: collection framework; design
Introduction

2
mps/design/config.txt
View file

@ -9,6 +9,7 @@ MPS Configuration
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: configuration; design
Introduction
@ -57,6 +58,7 @@ variety-reform_ branch. Client-specific customisation of the MPS will
be handled in source control, while the MPS source remains generic, to
reduce costs and increase reliability. See [RB_2012-09-13]_.
.. _variety-reform: /project/mps/branch/2012-08-15/variety-reform
Definitions

4
mps/design/critical-path.txt
View file

@ -8,6 +8,10 @@ The critical path through the MPS
:Date: 2012-09-07
:Revision: $Id$
:Copyright: See section `Copyright and License`_.
:Index terms:
single: critical path
single: path; critical
single: Memory Pool System; critical path
Introduction

1
mps/design/diag.txt
View file

@ -9,6 +9,7 @@ Diagnostic feedback
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: diagnostic feedback; design
Introduction

1
mps/design/finalize.txt
View file

@ -9,6 +9,7 @@ Finalization
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: finalization; design
Overview

1
mps/design/fix.txt
View file

@ -9,6 +9,7 @@ The generic fix function
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: fix function; design
Introduction

1
mps/design/guide.hex.trans.txt
View file

@ -10,6 +10,7 @@ Transliterating the alphabet into hexadecimal
:Date: 1997-04-11
:Revision: $Id$
:Copyright: See section `Copyright and License`_.
:Index terms: pair: hexadecimal; transliterating
Introduction

3
mps/design/guide.impl.c.format.txt
View file

@ -10,6 +10,9 @@ C Style -- formatting
:Format: rst
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: C language; formatting guide
pair: C language formatting; guide
Introduction

1
mps/design/interface-c.txt
View file

@ -9,6 +9,7 @@ C interface design
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: C interface; design
Introduction

1
mps/design/io.txt
View file

@ -9,6 +9,7 @@ I/O subsystem
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: I/O subsystem; design
Introduction

1
mps/design/keyword-arguments.txt
View file

@ -8,6 +8,7 @@ Keyword arguments in the MPS
:Date: 2013-05-09
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: keyword arguments; design
Introduction

1
mps/design/lib.txt
View file

@ -9,6 +9,7 @@ Library interface
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: library interface; design
Introduction

1
mps/design/lock.txt
View file

@ -9,6 +9,7 @@ The lock module
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: locking; design
Purpose

4
mps/design/locus.txt
View file

@ -9,6 +9,7 @@ MPS Configuration
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: locus manager; design
Introduction
@ -323,8 +324,7 @@ details, see implementation section and code, or user doc.
Loci
....
``Res LocusCreate(Locus *locusReturn, LocusAttrs attrs, ZoneGroup zg,
LocusAllocDesc adesc)``
``Res LocusCreate(Locus *locusReturn, LocusAttrs attrs, ZoneGroup zg, LocusAllocDesc adesc)``
_`.function.create`: A function to create a locus: ``adesc`` contains
the information about the allocation sequences in the locus, ``zg`` is

1
mps/design/message-gc.txt
View file

@ -9,6 +9,7 @@ GC messages
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: garbage collection messages; design
Introduction

3
mps/design/message.txt
View file

@ -9,6 +9,9 @@ Client message protocol
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: messages; design
single: client message protocol
Introduction

1
mps/design/object-debug.txt
View file

@ -9,6 +9,7 @@ Debugging features for client objects
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: debugging; design
Introduction

1
mps/design/pool.txt
View file

@ -9,6 +9,7 @@ Pool and pool class mechanisms
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: pool class mechanism; design
Definitions

3
mps/design/poolamc.txt
View file

@ -9,6 +9,9 @@ AMC pool class
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: AMC pool class; design
single: pool class; AMC design
Introduction

3
mps/design/poolams.txt
View file

@ -9,6 +9,9 @@ AMS pool class
:Status: draft design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: AMS pool class; design
single: pool class; AMS design
Introduction

3
mps/design/poolawl.txt
View file

@ -9,6 +9,9 @@ AWL pool class
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: AWL pool class; design
single: pool class; AWL design
Introduction

3
mps/design/poollo.txt
View file

@ -9,6 +9,9 @@ LO pool class
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: LO pool class; design
single: pool class; LO design
Introduction

3
mps/design/poolmfs.txt
View file

@ -9,6 +9,9 @@ MFS pool class
:Status: Incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: MFS pool class; design
single: pool class; MFS design
Overview

3
mps/design/poolmrg.txt
View file

@ -9,6 +9,9 @@ MRG pool class
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: MRG pool class; design
single: pool class; MRG design
Introduction

12
mps/design/poolmvff.txt
View file

@ -9,6 +9,9 @@ MVFF pool class
:Organization: Harlequin
:Status: incomplete doc
:Revision: $Id$
:Index terms:
pair: MVFF pool class; design
single: pool class; MVFF design
Introduction
@ -50,11 +53,11 @@ Methods
_`.method`: The MVFF pool supports the following methods:
``Res MVFFInit(Pool pool, va_list arg)``
``Res MVFFInit(Pool pool, Args arg)``
_`.method.init`: This takes six `keyword arguments`_:
.. _`keyword arguments`: keyword-arguments.rst
.. _`keyword arguments`: keyword-arguments
================================== ============================================
Keyword argument Description
@ -117,6 +120,7 @@ Implementation
--------------
_`.impl.free-list`: The pool stores its free list in a CBS (see
//gdr-peewit/info.ravenbrook.com/project/mps/branch/2013-05-17/emergency/design/poolmvff.txt
`design.mps.cbs <cbs/>`_), failing over in emergencies to a Freelist
(see design.mps.freelist) when the CBS cannot allocate new control
structures. This is the reason for the alignment restriction above.
@ -131,7 +135,9 @@ object size (in both cases we align up).
_`.design.seg-fail`: If allocating a segment fails, we try again with
a segment size just large enough for the object we're allocating. This
is in response to request.mps.170186.
is in response to `request.mps.170186`_.
.. _`request.mps.170186`: https://info.ravenbrook.com/project/mps/import/2001-11-05/mmprevol/request/mps/170186/
Document History

1
mps/design/prot.txt
View file

@ -9,6 +9,7 @@ The protection module
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: protection interface; design
Introduction

3
mps/design/protan.txt
View file

@ -9,6 +9,9 @@ ANSI implementation of protection module
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: ANSI; protection interface design
pair: ANSI protection interface; design
Introduction

3
mps/design/protli.txt
View file

@ -9,6 +9,9 @@ Linux implementation of protection module
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: Linux; protection interface design
pair: Linux protection interface; design
Introduction

1
mps/design/protocol.txt
View file

@ -9,6 +9,7 @@ Protocol inheritance
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: protocol inheritance; design
Introduction

4
mps/design/protsu.txt
View file

@ -9,6 +9,10 @@ SunOS 4 protection module
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: SunOS 4; protection interface design
pair: SunOS 4 protection interface; design
.. warning::

1
mps/design/pthreadext.txt
View file

@ -9,6 +9,7 @@ POSIX thread extensions
:Status: Draft document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: POSIX thread extensions; design
Introduction

1
mps/design/reservoir.txt
View file

@ -9,6 +9,7 @@ The low-memory reservoir
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: reservoir; design
Introduction

1
mps/design/ring.txt
View file

@ -9,6 +9,7 @@ Ring data structure
:Status: incomplete design
:Revision: $Id$
:Copyright: See section `Copyright and License`_.
:Index terms: pair: ring structure; design
Introduction

1
mps/design/root.txt
View file

@ -9,6 +9,7 @@ Root manager
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: root manager; design
Basics

1
mps/design/scan.txt
View file

@ -9,6 +9,7 @@ The generic scanner
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: generic scanner; design
Summaries

1
mps/design/seg.txt
View file

@ -9,6 +9,7 @@ Segment data structure
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: segments; design
Introduction

21
mps/design/shield.txt
View file

@ -9,6 +9,7 @@ Shield
:Status: incomplete guide
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: shield; design
Introduction
@ -46,13 +47,13 @@ by MPS).
``void ShieldRaise(Arena arena, Seg seg, AccessSet mode)``
Prevent the mutator accessing the memory in the specified mode
(``AccessREAD``, ``AccessWRITE``, or both).
Prevent the mutator accessing the memory in the specified mode
(``AccessREAD``, ``AccessWRITE``, or both).
``void ShieldLower(Arena arena, Seg seg, AccessSet mode)``
Allow the mutator to access the memory in the specified mode
(``AccessREAD``, ``AccessWRITE``, or both).
Allow the mutator to access the memory in the specified mode
(``AccessREAD``, ``AccessWRITE``, or both).
If the mutator attempts an access that hits a shield, the MPS gets a
barrier hit (in the form of a fault, interrupt, exception), quickly
@ -65,11 +66,11 @@ this.
``void ShieldSuspend(Arena arena)``
Stop all registered mutator threads.
Stop all registered mutator threads.
``void ShieldResume(Arena arena)``
Resume all registered mutator threads.
Resume all registered mutator threads.
Control of collector access
@ -108,7 +109,7 @@ calls to ``ShieldExpose()`` on the same segment, as long as each is
balanced by a corresponding ``ShieldCover()`` before ``ShieldLeave()``
is called). A usage count is maintained on each segment in
``seg->depth``: a positive "depth" means a positive number of
outstanding reasons why the segment must be exposed to the collector.
outstanding *reasons* why the segment must be exposed to the collector.
When the usage count reaches zero, there is no longer any reason the
segment should be unprotected, and the Shield could re-instate
hardware protection.
@ -116,14 +117,14 @@ hardware protection.
However, as a performance-improving hysteresis, the Shield defers
re-protection, maintaining a cache of the last ``ShieldCacheSIZE``
times a segment no longer had a reason to be collector-accessible.
Presence in the cache counts as a 'reason': segments in the cache have
Presence in the cache counts as a reason: segments in the cache have
``seg->depth`` increased by one. As segments get pushed out of the
cache, or at ``ShieldLeave()``, this artificial 'reason' is
cache, or at ``ShieldLeave()``, this artificial reason is
decremented from ``seg->depth``, and (if ``seg->depth`` is now zero)
the deferred reinstatement of hardware protection happens.
So whenever hardware protection is temporarily removed to allow
collector access, there is a 'nurse' that will ensure this protection
collector access, there is a *nurse* that will ensure this protection
is re-established: the nurse is either the balancing ``ShieldCover()``
call in collector code, or an entry in the shield cache.

20
mps/design/sig.txt
View file

@ -9,6 +9,9 @@ Signatures in the MPS
:Date: 2013-05-09
:Revision: $Id$
:Copyright: See section `Copyright and License`_.
:Index terms:
pair: structure signatures; design
single: signatures
Introduction
@ -24,13 +27,15 @@ defects early.
Overview
--------
Signatures are magic numbers which are written into structures when
Signatures are `magic numbers`_ which are written into structures when
they are created and invalidated (by overwriting with ``SigInvalid``)
when they are destroyed. They provide a limited form of run-time type
checking and dynamic scope checking. They are a simplified form of
"Structure Marking", a technique used in the Multics filesystem
[THVV_1995]_.
.. _`magic numbers`: http://en.wikipedia.org/wiki/Magic_number_(programming)
Definitions
-----------
@ -54,7 +59,7 @@ This is a 32-bit hex constant, spelled according to guide.hex.trans_::
ABCDEFGHIJKLMNOPQRSTUVWXYZ
ABCDEF9811C7340BC6520F3812
.. _guide.hex.trans: ./guide.hex.trans.txt
.. _guide.hex.trans: guide.hex.trans
This allows the structure to be recognised when looking at memory in a hex
dump or memory window, or found using memory searches.
@ -97,10 +102,13 @@ Do not do anything else with signatures. See `.rule.purpose`_.
Checking
--------
The signature is checked in various ways. Every function that takes a
(pointer to) a signed structure should check its argument using the ``AVERT``
macro. This macro has different definitions depending on how the MPS is
compiled. It may simply check the signature directly, or call the full
checking function for the structure.
(pointer to) a signed structure should check its argument using the
``AVERT`` macro. This macro has different definitions depending on how
the MPS is compiled (see design.mps.config.def.var_). It may simply check
the signature directly, or call the full checking function for the
structure.
.. _design.mps.config.def.var: config#def-var
The checking function for the structure should also validate the
signature as its first step using the ``CHECKS()`` macro (see

1
mps/design/splay.txt
View file

@ -9,6 +9,7 @@ Splay trees
:Status: draft document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: splay trees; design
Introduction

3
mps/design/sso1al.txt
View file

@ -9,6 +9,9 @@ Stack scanner for Digital Unix on Alpha
:Status: draft document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: Digital Unix on Alpha stack scanner; design
pair: Digital Unix on Alpha; stack scanner design
.. warning::

462
mps/design/strategy.txt Normal file
View file

@ -0,0 +1,462 @@
.. mode: -*- rst -*-
MPS Strategy
============
:Tag: design.mps.strategy
:Author: Nick Barnes
:Organization: Ravenbrook Limited
:Date: 2013-06-04
:Revision: $Id$
:Copyright: See section `Copyright and License`_.
Introduction
------------
_`.intro` This is the design of collection strategy for the MPS.
_`.readership` MPS developers.
Overview
--------
_`.overview` The MPS uses "strategy" code to make three decisions:
- when to start a collection trace;
- what to condemn;
- how to schedule tracing work.
This document describes the current strategy, identifies some
weaknesses in it, and outlines some possible future development
directions.
Requirements
------------
[TODO: source some from req.dylan, or do an up-to-date requirements
analysis -- NB 2013-03-25]
Garbage collection is a trade-off between time and space: it consumes
some [CPU] time in order to save some [memory] space. Strategy shifts
the balance point. A better strategy will take less time to produce
more space. Examples of good strategy might include:
- choosing segments to condemn which contain high proportions of dead
objects;
- starting a trace when a large number of objects have just died;
- doing enough collection soon enough that the client program never
suffers low-memory problems;
- using otherwise-idle CPU resources for tracing.
Conversely, it would be bad strategy to do the reverse of each of
these (condemning live objects; tracing when there's very little
garbage; not collecting enough; tracing when the client program is
busy).
Abstracting from these notions, requirements on strategy would
relate to:
- Maximum pause time and other utilization metrics (for example,
bounded mutator utilization, minimum mutator utilization, total MPS
CPU usage);
- Collecting enough garbage (for example: overall heap size;
low-memory requirements).
- Allowing client control (for example, client recommendations for
collection timing or condemnation).
There are other possible strategy considerations which are so far
outside the scope of current strategy and MPS design that this
document disregards them. For example, either inferring or allowing
the client to specify preferred relative object locations ("this
object should be kept in the same cache line as that one"), to improve
cache locality.
Generations
-----------
The largest part of the current MPS strategy implementation is the
support for generational GC. Generations are only fully supported for
AMC (and AMCZ) pools. See under "Non-AMC Pools", below, for more
information.
Data Structures
...............
The fundamental structure of generational GC is the ``Chain``,
which describes a set of generations. A chain is created by client
code calling ``mps_chain_create()``, specifying the "size" and
"mortality" for each generation. When creating an AMC pool, the
client code must specify the chain which will control collections for
that pool. The same chain may be used for multiple pools.
Each generation in a chain has a ``GenDesc`` structure,
allocated in an array pointed to from the chain. Each AMC pool has a
set of ``PoolGen`` structures, one per generation. The PoolGens
for each generation point to the GenDesc and are linked together in a
ring on the GenDesc. These structures are (solely?) used to gather
information for strategy decisions.
The arena has a unique ``GenDesc`` structure, named
``topGen`` and described in comments as "the dynamic generation"
(although in fact it is the *least* dynamic generation). Each AMC
pool has one more PoolGen than there are GenDescs in the chain. The
extra PoolGen refers to this topGen.
AMC segments have a segment descriptor ``amcSegStruct`` which is
a ``GCSegStruct`` with two additional fields. One field
``segTypeP`` is a pointer either to the per-generation per-pool
``amcGen`` structure (a subclass of ``PoolGen``), or to a
nailboard (which then points to an amcGen). The other field
``new`` is a boolean used for keeping track of memory usage for
strategy reasons (see below under 'Accounting'). The ``amcGen``
is used for statistics (``->segs``) and forwarding buffers
(``->forward``).
The AMC pool class only ever allocates a segment in order to fill a
buffer: either the buffer for a client Allocation Point, or a
forwarding buffer. In order to support generational collection, there
is a subclass ``amcBuf`` of ``SegBuf``, with a
``gen`` field (pointing to a ``amcGen``). So in
``AMCBufferFill()`` the generation of the new segment can be
determined.
When an AMC pool is created, these ``amcGen`` and
``amcBuf`` structures are all created, and the
``amcBuf->gen`` fields initialized so that the forwarding buffer
of each amcGen knows that it belongs to the next "older" amcGen (apart
from the "oldest" amcGen - that which refers to the topGen - whose
forwarding buffer belongs to itself).
When copying an object in ``AMCFix()``, the object's current
generation is determined (``amcSegGen()``), and the object is
copied to that amcGen's forwarding buffer, using the buffer protocol.
Thus, objects are "promoted" up the chain of generations until they
end up in the topGen, which is shared between all chains and all
pools.
For statistics and reporting purposes, when ``STATISTICS`` is
on, each AMC pool has an array of ``PageRetStruct``s, one per
trace. This structure has many ``Count`` fields, and is
intended to help to assess AMC page retention code. See job001811.
Zones
.....
All collections in the MPS start with condemnation of a complete
``ZoneSet``. Each generation in each chain has a zoneset
associated with it (``chain->gen[N].zones``); the condemned
zoneset is the union of some number of generation's zonesets. It is
condemned by code in the chain system calling
``TraceCondemnZones()``. This is either for all chains
(``ChainCondemnAll()`` called for every chain from
``traceCondemnAll()``) or for some number of generations in a
single chain (``ChainCondemnAuto()`` called from
``TracePoll()``). Note that the condemnation is of every
automatic-pool segment in any zone in the zoneset. It is not limited
to the segments actually associated with the condemned generation(s).
An attempt is made to use distinct zonesets for different generations.
Whenever a segment is allocated (``AMCBufferFill()``), a
``SegPref`` is created containing the generation number
(obtained from ``amcBuf->gen->pgen->nr``) and passed to
``SegAlloc()``. The arena keeps a zoneset for each generation
number (up to ``VMArenaGenCount``, defined in
``arenavm.c`` to be ``MPS_WORD_WIDTH/2``), and a
``freeSet``. The zoneset for each generation number starts out
empty, and the ``freeSet`` starts out ``ZoneSetUNIV``.
When a segment is allocated with a ``SegPref`` with a generation
number, an attempt is made to allocate it in the corresponding zoneset
(``pagesFindFreeInZones()``). If the zoneset is empty, an
attempt is made to allocate it in the ``freeSet`` zoneset.
After it is allocated, the zones it occupies are removed from the
``freeSet`` and (if there's a generation ``SegPref``)
added to the zoneset for that generation number.
Note that this zone placement code knows nothing of chains,
generations, pool classes, etc. It is based solely on the generation
*number*, so generations with the same number from different chains
share a zoneset preference for the purpose of placing newly allocated
segments. Combined with the fact that condemnation is per-zone, this
effectively means that generations in distinct chains are collected
together. One consequence of this is that we don't have a very fine
granularity of control over collection: a garbage collection of all
chains together is triggered by the most eager chain. There's no way
for a library or other small part of a client program to arrange
independent collection of a separate pool or chain.
When ``AMCBufferFill()`` gets the allocated segment back, it
adds it to the zoneset associated with that generation in the pool's
controlling chain. Note that a chain's per-generation zonesets, which
represent the zones in which segments for that generation in that
chain have been placed, are quite distinct from the arena-wide
per-generation-number zonesets, which represent the zones in which
segments for that generation number in any chain have been placed.
The arena-wide per-generation-number zoneset
``vmArena->genZoneSet[N]`` is augmented in
``vmAllocComm()``. The per-chain per-generation zoneset
``chain->gen[N].zones`` is augmented in
``PoolGenUpdateZones()``. Neither kind of zoneset can ever
shrink.
Accounting
..........
- ``gen[N].mortality``
- Specified by the client.
- TODO: fill in how this is used.
- ``gen[N].capacity``
- Specified by the client.
- TODO: fill in how this is used.
- ``amcSeg->new``
- TODO: fill this in
- ``pgen->totalSize``:
- incremented by ``AMCBufferFill()``;
- decremented by ``amcReclaimNailed()`` and ``AMCReclaim()``;
- added up by ``GenDescTotalSize(gen)``.
- ``pgen->newSize``:
- incremented by ``AMCBufferFill()`` (*when not ramping*) and ``AMCRampEnd()``;
- decremented by ``AMCWhiten()``,
- added up by ``GenDescNewSize(gen)``.
- ``gen[N].proflow``:
- set to 1.0 by ``ChainCreate()``;
- ``arena->topGen.proflow`` set to 0.0 by ``LocusInit(arena)``;
- *The value of this field is never used*.
- ``pgen->newSizeAtCreate``:
- set by ``traceCopySizes()`` (that is its purpose);
- output by ``TraceStartGenDesc_diag()``.
Ramps
.....
The intended semantics of ramping are pretty simple. It allows the
client to advise us of periods of large short-lived allocation on a
particular AP. Stuff allocated using that AP during its "ramp" will
probably be dead when the ramp finishes. How the MPS makes use of this
advice is up to us, but for instance we might segregate those objects,
collect them less enthusiastically during the ramp and then more
enthusiastically soon after the ramp finishes. Ramps can nest.
A ramp is entered by calling::
mps_ap_alloc_pattern_begin(ap, mps_alloc_pattern_ramp())
or similar, and left in a similar way.
This is implemented on a per-pool basis, for AMC only (it's ignored by
the other automatic pools). PoolAMC throws away the identity of the AP
specified by the client. The implementation is intended to work by
changing the generational forwarding behaviour, so that there is a "ramp
generation" - one of the regular AMC generations - which forwards to
itself if collected during a ramp (instead of promoting to an older
generation). It also tweaks the strategy calculation code, in a way
with consequences I am documenting elsewhere.
Right now, the code sets this ramp generation to the last generation
specified in the pool's "chain": it ordinarily forwards to the
"after-ramp" generation, which is the "dynamic generation" (i.e. the
least dynamic generation, i.e. the arena-wide "top generation"). My
recollection, and some mentions in design/poolamc, suggests that the
ramp generation used to be chosen differently from this.
So far, it doesn't sound too ghastly, I guess, although the subversion
of the generational system seems a little daft. Read on....
An AMC pool has a ``rampMode`` (which is really a state of a state
machine), taking one of five values: OUTSIDE, BEGIN, RAMPING, FINISH,
and COLLECTING (actually the enum values are called RampX for these
X). We initialize in OUTSIDE. The pool also has a ``rampCount``,
which is the ramp nesting depth and is used to allow us to ignore ramp
transitions other than the outermost. According to design/poolamc,
there's an invariant (in BEGIN or RAMPING, ``rampCount > 0``; in
COLLECTING or OUTSIDE, ``rampCount == 0``), but this isn't checked in
``AMCCheck()`` and in fact is false for COLLECTING (see below).
There is a small set of events causing state machine transitions:
- entering an outermost ramp;
- leaving an outermost ramp;
- condemning any segment of a ramp generation (detected in AMCWhiten);
- reclaiming any AMC segment.
Here's pseudo-code for all the transition events:
Entering an outermost ramp:
if not FINISH, go to BEGIN.
Leaving an outermost ramp:
if RAMPING, go to FINISH. Otherwise, go to OUTSIDE.
Condemning a ramp generation segment:
If BEGIN, go to RAMPING and make the ramp generation forward
to itself (detach the forwarding buffer and reset its generation).
If FINISH, go to COLLECTING and make the ramp generation
forward to the after-ramp generation.
Reclaiming any AMC segment:
If COLLECTING:
if ``rampCount > 0``, go to BEGIN. Otherwise go to OUTSIDE.
Now, some deductions:
1. When OUTSIDE, the count is always zero, because (a) it starts that
way, and the only ways to go OUTSIDE are (b) by leaving an outermost
ramp (count goes to zero) or (c) by reclaiming when the count is zero.
2. When BEGIN, the count is never zero (consider the transitions to
BEGIN and the transition to zero).
3. When RAMPING, the count is never zero (again consider transitions to
RAMPING and the transition to zero).
4. When FINISH, the count can be anything (the transition to FINISH has
zero count, but the Enter transition when FINISH can change that and
then it can increment to any value).
5. When COLLECTING, the count can be anything (from the previous fact,
and the transition to COLLECTING).
6. *This is a bug!!* The ramp generation is not always reset (to forward
to the after-ramp generation). If we get into FINISH and then see
another ramp before the next condemnation of the ramp generation, we
will Enter followed by Leave. The Enter will keep us in FINISH, and
the Leave will take us back to OUTSIDE, skipping the transition to the
COLLECTING state which is what resets the ramp generation forwarding
buffer. [TODO: check whether I made an issue and/or fixed it; NB 2013-06-04]
The simplest change to fix this is to change the behaviour of the Leave
transition, which should only take us OUTSIDE if we are in BEGIN or
COLLECTING. We should also update design/poolamc to tell the truth, and
check the invariants, which will be these:
OUTSIDE => zero
BEGIN => non-zero
RAMPING => non-zero
A cleverer change might radically rearrange the state machine
(e.g. reduce the number of states to three) but that would require
closer design thought and should probably be postponed until we have a
clearer overall strategy plan.
While I'm writing pseudo-code versions of ramp-related code, I should
mention this other snippet, which is the only other code relating to
ramping (these notes are useful when thinking about the broader strategy
code):
In ``AMCBufferFill()``, if we're RAMPING, and filling the forwarding
buffer of the ramp generation, and the ramp generation is the
forwarding buffer's generation, set ``amcSeg->new`` to FALSE. Otherwise,
add the segment size to ``poolGen.newSize``.
And since I've now mentioned the ``amcSeg->new`` flag, here are the only
other uses of that:
- it initializes as TRUE.
- When leaving an outermost ramp, go through all the segments in the
pool. Any non-white segment in the rampGen with new set to FALSE has
its size added to ``poolGen->newSize`` and gets new set to TRUE.
- in ``AMCWhiten()``, if new is TRUE, the segment size is deducted
from ``poolGen.newSize`` and new is set to FALSE.
Non-AMC Pools
.............
The implementations of AMS, AWL, and LO pool classes are all aware of
generations (this is necessary because all tracing is driven by the
generational data structures described above), but do not make use of
them. For LO and AWL, when a pool is created, a chain with a single
generation is also created, with size and mortality parameters
hard-wired into the pool-creation function (LOInit, AWLInit). For
AMS, a chain is passed as a pool creation parameter into
``mps_pool_create()``, but this chain must also have only a
single generation (otherwise ``ResPARAM`` is returned).
Note that these chains are separate from any chain used by an AMC pool
(except in the trivial case when a single-generation chain is used for
both AMC and AMS). Note also that these pools do not use or point to
the ``arena->topGen``, which applies only to AMC.
Non-AMC pools have no support for ramps.
Starting a Trace
................
TODO: Why do we start a trace? How do we choose what to condemn?
Trace Progress
..............
TODO: When do we do some tracing work? How much tracing work do we do?
Document History
----------------
- 2013-06-04 NB Checked this in although it's far from complete.
Pasted in my 'ramping notes' from email, which mention some bugs
which I may have fixed (TODO: check this).
.. _NB: http://www.ravenbrook.com/consultants/nb/
Copyright and License
---------------------
Copyright © 2013 Ravenbrook Limited. All rights reserved.
<http://www.ravenbrook.com/>. This is an open source license. Contact
Ravenbrook for commercial licensing options.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Redistributions in any form must be accompanied by information on how
to obtain complete source code for this software and any
accompanying software that uses this software. The source code must
either be included in the distribution or be available for no more than
the cost of distribution plus a nominal fee, and must be freely
redistributable under reasonable conditions. For an executable file,
complete source code means the source code for all modules it contains.
It does not include source code for modules or files that typically
accompany the major components of the operating system on which the
executable file runs.
**This software is provided by the copyright holders and contributors
"as is" and any express or implied warranties, including, but not
limited to, the implied warranties of merchantability, fitness for a
particular purpose, or non-infringement, are disclaimed. In no event
shall the copyright holders and contributors be liable for any direct,
indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or
services; loss of use, data, or profits; or business interruption)
however caused and on any theory of liability, whether in contract,
strict liability, or tort (including negligence or otherwise) arising in
any way out of the use of this software, even if advised of the
possibility of such damage.**

1
mps/design/telemetry.txt
View file

@ -9,6 +9,7 @@ Telemetry
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: telemetry; design
Introduction

1
mps/design/thread-manager.txt
View file

@ -9,6 +9,7 @@ Thread Manager
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: thread manager; design
Purpose

1
mps/design/thread-safety.txt
View file

@ -9,6 +9,7 @@ Thread safety in the MPS
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: thread safety; design
Introduction

1
mps/design/trace.txt
View file

@ -9,6 +9,7 @@ Tracer
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: tracer; design
Introduction

1
mps/design/type.txt
View file

@ -9,6 +9,7 @@ General MPS types
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: general types; design
Introduction

1
mps/design/version-library.txt
View file

@ -9,6 +9,7 @@ Library version mechanism
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: library version mechanism; design
Introduction

3
mps/design/version.txt
View file

@ -9,6 +9,9 @@ Software versions
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms:
pair: software versions; design
single: versions; design
Introduction

1
mps/design/vm.txt
View file

@ -9,6 +9,7 @@ Virtual mapping
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: virtual mapping; design
Introduction

2
mps/design/vman.txt
View file

@ -9,6 +9,8 @@ ANSI fake VM
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: ANSI fake VM; design
_`.intro`: The ANSI fake VM is an implementation of the MPS VM
interface (see design.mps.vm) using services provided by the ANSI C

2
mps/design/vmo1.txt
View file

@ -9,6 +9,8 @@ VM for Digital Unix
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: VM for Digital Unix; design
.. warning::

1
mps/design/vmso.txt
View file

@ -9,6 +9,7 @@ VM for Solaris
:Status: incomplete document
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: VM for Solaris; design
.. warning::

11
mps/design/writef.txt
View file

@ -9,6 +9,7 @@ The WriteF function
:Status: incomplete design
:Revision: $Id$
:Copyright: See `Copyright and License`_.
:Index terms: pair: WriteF function; design
Introduction
@ -82,7 +83,7 @@ Code Bame Type Example rendering
======= =========== ================== ======================================
``$A`` address ``Addr`` ``000000019EF60010``
``$P`` pointer ``void *`` ``000000019EF60100``
``$F`` function ``void *(*)()`` ``000000019EF60100``
``$F`` function ``void *(*)()`` ``0001D69E01000000`` (see `.f`_)
``$S`` string ``char *`` ``hello``
``$C`` character ``char`` ``x``
``$W`` word ``ULongest`` ``0000000000109AE0``
@ -99,10 +100,10 @@ incredible snazzy output engine. We only need it for ``Describe()``
methods and assertion messages. At the moment it's a very simple bit
of code -- let's keep it that way.
_`.f`: The ``F`` code is used for function pointers. They are
currently printed as a hexadecimal string of the appropriate length
for the platform, and may one day be extended to include function name
lookup.
_`.f`: The ``F`` code is used for function pointers. ISO C forbids casting
function pointers to other types, so the bytes of their representation are
written sequentially, and may have a different endianness to other pointers.
Could be smarter, or even look up function names, but see `.snazzy`_.
Document History

3
mps/manual/Makefile
View file

@ -40,7 +40,8 @@ help:
@echo " tools to install a local copy of the Python tools using virtualenv"
clean:
-rm -rf $(BUILDDIR)/{changes,devhelp,dirhtml,doctest,doctrees,epub,html,htmlhelp,json,latex,linkcheck,locale,man,pickle,qthelp,singlehtml,texinfo,text,converted}
-rm -rf $(BUILDDIR)/{changes,devhelp,dirhtml,doctest,doctrees,epub,html,htmlhelp,json,latex,linkcheck,locale,man,pickle,qthelp,singlehtml,texinfo,text}
-find $(BUILDDIR)/source/design -name '*.rst' ! -name 'index.rst' ! -name 'old.rst' -exec rm -f '{}' ';'
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html

459
mps/manual/html/_sources/design/alloc-frame.txt Normal file
View file

@ -0,0 +1,459 @@
.. _design-alloc-frame:
.. index::
pair: allocation frames; design
Allocation frame protocol
=========================
.. mps:prefix:: design.mps.alloc-frame
Introduction
------------
:mps:tag:`intro` This document explains the design of the support for
allocation frames in MPS.
:mps:tag:`readership` This document is intended for any MM developer.
:mps:tag:`overview` Allocation frames are used for implementing stack pools;
each stack frame corresponds to an allocation frame. Allocation frames
may also be suitable for implementing other sub-pool groupings, such
as generations and ramp allocation patterns.
:mps:tag:`overview.ambition` We now believe this to be a design that loses
too many advantages of stack allocation for questionable gains. The
requirements are almost entirely based on unanalysed anecdote, instead
of actual clients.
.. note::
We plan to supersede this with a stack pool design at some point
in the future. Pekka P. Pirinen, 2000-03-09.
Definitions
-----------
:mps:tag:`def.alloc-frame` An allocation frame is a generic name for a
device which groups objects together with other objects at allocation
time, and which may have a parent/child relationship with other
allocation frames.
Purpose
-------
:mps:tag:`purpose.stack-allocation` The allocation frame protocol is
intended to support efficient memory management for stack allocation,
that is, the allocation of objects which have dynamic extent.
:mps:tag:`purpose.general` The allocation frame protocol is intended to be
sufficiently general that it will be useful in supporting other types
of nested allocation patterns too. For example, it could be used to
for EPVM-style save and restore, ramp allocation patterns or
generations.
Requirements
------------
Known requirements
..................
:mps:tag:`req.stack-alloc` Provide a interface for clients to describe a
stack allocation pattern, as an alternative to using the control
stack.
:mps:tag:`req.efficient` Permit an implementation which is comparable in
efficiency to allocating on the control stack.
:mps:tag:`req.ap` Support allocation via allocation points (APs).
:mps:tag:`req.format` Support the allocation of formatted objects.
:mps:tag:`req.scan` Ensure that objects in allocation frames can participate
in garbage collection by being scanned.
:mps:tag:`req.fix` Ensure that objects in allocation frames can participate
in garbage collection by accepting Fix requests.
:mps:tag:`req.condemn` Ensure that objects in allocation frames can
participate in garbage collection by being condemned.
:mps:tag:`attr.locking` Minimize the synchronization cost for the creation
and destruction of frames.
Proto-requirements
..................
:mps:tag:`proto-req` The following are possible requirements that might be
important in the future. The design does not necessarily meet all
these requirements, but it does consider them all. Each requirement
either has direct support in the framework, or could be supported with
future additions to the framework.
:mps:tag:`req.parallels` The allocation frame protocol should provide a
framework for exploiting the parallels between stack extents,
generations and "ramps".
:mps:tag:`req.pool-destroy` It should be possible to use allocation frames
to free all objects in a pool without destroying the pool.
:mps:tag:`req.epvm` It should be possible to implement EPVM-style save and
restore operations by creating and destroying allocation frames.
:mps:tag:`req.subst` It should be possible to substitute a stack pool with a
GC-ed pool so that erroneous use of a stack pool can be detected.
:mps:tag:`req.format-extensions` It should be possible for stack pools to
utilize the same format as any other pool, including debugging formats
that include fenceposting, etc.
:mps:tag:`req.mis-nest` Should ensure "mis-nested" stacks are safe.
:mps:tag:`req.non-top-level` Should support allocation in the non-top stack
extent.
:mps:tag:`req.copy-if-necessary` Should ensure that stack pools can support
"copy-if-necessary" (so that low-level system code can heapify stack
objects.)
:mps:tag:`req.preserve` When an object is in an allocation frame which is
being destroyed, it should be possible to preserve that object in the
parent frame.
:mps:tag:`req.contained` Should allow clients to ask if an object is
"contained" in a frame. The object is contained in a frame if it is
affected when the frame is ended.
:mps:tag:`req.alloc-with-other` Should allow clients to allocate an object
in the same frame as another object.
Overview
--------
:mps:tag:`frame-classes` The protocol supports different types of allocation
frames, which are represented as "frame classes". It's up to pools to
determine which classes of allocation frames they support. Pools which
support more than one frame class rely on the client to indicate which
class is currently of interest. The client indicates this by means of
an operation which stores the class in the buffer to which the
allocation point is attached.
:mps:tag:`frame-handles` Allocation frames are described via abstract "frame
handles". Pools may choose what the representation of a frame handle
should be. Frame handles are static, and the client need not store
them in a GC root.
:mps:tag:`lightweight-frames` The design includes an extension to the
allocation point protocol, which permits the creation and destruction
of allocation frames without the necessity for claiming the arena
lock. Such frames are called "lightweight frames".
Operations
----------
:mps:tag:`op.intro` Each operation has both an external (client) interface
and an internal (MPS) interface. The external function takes an
allocation point as a parameter, determines which buffer and pool it
belongs to, and calls the internal function with the buffer and pool
as parameters.
:mps:tag:`op.obligatory` The following operations are supported on any
allocation point which supports allocation frames:-
:mps:tag:`operation.push` The :c:func:`PushFrame()` operation creates a new
allocation frame of the currently chosen frame class, makes this new
frame the current frame, and returns a handle for the frame.
:mps:tag:`operation.pop` The :c:func:`PopFrame()` operation takes a frame handle
as a parameter. Some pool classes might insist or assume that this is
the handle for the current frame. It finds the parent of that frame
and makes it the current frame. The operation indicates that all
children of the new current frame contain objects which are likely to
be dead. The reclaim policy is up to the pool; some classes might
insist or assume that the objects must be dead, and eagerly free them.
Note that this might introduce the possibility of leaving dangling
pointers elsewhere in the arena. If so, it's up to the pool to decide
what to do about this.
:mps:tag:`op.optional` The following operations are supported for some
allocation frames, but not all. Pools may choose to support some or
all of these operations for certain frame classes. An unsupported
operation will return a failure value:-
:mps:tag:`operation.select` The :c:func:`SelectFrame()` operation takes a frame
handle as a parameter and makes that frame the current frame. It does
not indicate that any children of the current frame contain objects
which are likely to be dead.
:mps:tag:`operation.select-addr` The :c:func:`SelectFrameOfAddr()` operation takes
an address as a parameter and makes the frame of that address the
current frame. It does not indicate that any children of the current
frame contain objects which are likely to be dead.
:mps:tag:`operation.in-frame` The :c:func:`AddrInFrame()` operation determines
whether the supplied address is the address of an object allocated in
the supplied frame, or any child of that frame.
:mps:tag:`operation.set` The :c:func:`SetFrameClass()` operation takes a frame
class and an allocation point as parameters, and makes that the
current frame class for the allocation point. The next :c:func:`PushFrame()`
operation will create a new frame of that class.
Interface
---------
External types
..............
:mps:tag:`type.client.frame-handle` Frame handles are defined as the abstract
type :c:type:`mps_frame_t`.
.. c:type:: struct mps_frame_class_s *mps_frame_class_t
:mps:tag:`type.client.frame-class` Frame classes are defined as an abstract
type.
:mps:tag:`type.client.frame-class.access` Clients access frame classes by
means of dedicated functions for each frame class.
External functions
..................
:mps:tag:`fn.client.push` :c:func:`mps_ap_frame_push()` is used by clients to
invoke the :c:func:`PushFrame()` operation. For lightweight frames, this
might not invoke the corresponding internal function.
:mps:tag:`fn.client.pop` :c:func:`mps_ap_frame_pop()` is used by clients to invoke
the :c:func:`PopFrame()` operation. For lightweight frames, this might not
invoke the corresponding internal function.
.. c:function:: mps_res_t mps_ap_frame_select(mps_ap_t buf, mps_frame_t frame)
:mps:tag:`fn.client.select` This following function is used by clients to
invoke the :c:func:`SelectFrame()` operation.
.. c:function:: mps_res_t mps_ap_frame_select_from_addr(mps_ap_t buf, mps_addr_t addr)
:mps:tag:`fn.client.select-addr` This function is used by clients to invoke
the :c:func:`SelectFrameOfAddr()` operation.
.. c:function:: mps_res_t mps_ap_addr_in_frame(mps_bool_t *inframe_o, mps_ap_t buf, mps_addr_t *addrref, mps_frame_t frame)
:mps:tag:`fn.client.in-frame` This function is used by clients to invoke the
:c:func:`AddrInFrame()` operation.
.. c:function:: mps_res_t mps_ap_set_frame_class(mps_ap_t buf, mps_frame_class_t class)
:mps:tag:`fn.client.set` This function is used by clients to invoke the
:c:func:`SetFrameClass()` operation.
.. c:function:: mps_frame_class_t mps_alloc_frame_class_stack(void)
:mps:tag:`fn.client.stack-frame-class` This function is used by clients to
access the frame class used for simple stack allocation.
Internal types
..............
.. c:type:: struct AllocFrameStruct *AllocFrame
:mps:tag:`type.frame-handle` Frame handles are defined as an abstract type.
.. c:type:: struct AllocFrameClassStruct *AllocFrameClass
:mps:tag:`type.frame-class` Frame classes are defined as an abstract type.
.. c:type:: Res (*PoolFramePushMethod)(AllocFrame *frameReturn, Pool pool, Buffer buf)
:mps:tag:`fn.push` A pool method of this type is called (if needed) to
invoke the :c:func:`PushFrame()` operation.
.. c:type:: Res (*PoolFramePopMethod)(Pool pool, Buffer buf, AllocFrame frame)
:mps:tag:`fn.pop` A pool method of this type is called (if needed)
to invoke the PopFrame operation:
.. c:type:: Res (*PoolFrameSelectMethod)(Pool pool, Buffer buf, AllocFrame frame)
:mps:tag:`fn.select` A pool method of this type is called to invoke the
:c:func:`SelectFrame()` operation.
.. c:type:: Res (*PoolFrameSelectFromAddrMethod)(Pool pool, Buffer buf, Addr addr)
:mps:tag:`fn.select-addr` A pool method of this type is called to invoke the
:c:func:`SelectFrameOfAddr()` operation.
.. c:type:: Res (*PoolAddrInFrameMethod)(Bool *inframeReturn, Pool pool, Seg seg, Addr *addrref, AllocFrame frame)
:mps:tag:`fn.in-frame` A pool method of this type is called to invoke the
:c:func:`AddrInFrame()` operation.
.. c:type:: Res (*PoolSetFrameClassMethod)(Pool pool, Buffer buf, AllocFrameClass class)
:mps:tag:`fn.set` A pool method of this type is called to invoke the
:c:func:`SetFrameClass()` operation.
Lightweight frames
-------------------
Overview
........
:mps:tag:`lw-frame.overview` Allocation points provide direct support for
lightweight frames, and are designed to permit PushFrame and PopFrame
operations without the need for locking and delegation to the pool
method. Pools can disable this mechanism for any allocation point, so
that the pool method is always called. The pool method will be called
whenever synchronization is required for other reasons (e.g. the
buffer is tripped).
:mps:tag:`lw-frame.model` Lightweight frames offer direct support for a
particular model of allocation frame use, whereby the PushFrame
operation returns the current allocation pointer as a frame handle,
and the PopFrame operation causes the allocation pointer to be reset
to the address of the frame handle. This model should be suitable for
simple stack frames, where more advanced operations like SelectFrame
are not supported. It may also be suitable for more advanced
allocation frame models when they are being used simply. The use of a
complex operation always involves synchronization via locking, and the
pool may disable lightweight synchronization temporarily at this time.
State
.....
:mps:tag:`lw-frame.states` Allocation points supporting lightweight frames
will be in one of the following states:
============ ================================================================
Valid Indicates that :c:func:`PushFrame()` can be a lightweight
operation and need not be synchronized.
PopPending Indicates that there has been a :c:func:`PopFrame()` operation
that the pool must respond to.
Disabled Indicates that the pool has disabled support for lightweight
operations for this AP.
============ ================================================================
These states are in addition to the state normally held by an AP for
allocation purposes. An AP will be in the Disabled state at creation.
:mps:tag:`lw-frame.transitions` State transitions happen under the following
circumstances:
======================= ====================================================
Valid → PopPending As a result of a client :c:func:`PopFrame()`
operation.
Valid → Disabled At the choice of the pool (for example, when
responding to a :c:func:`SelectFrame()` operation).
PopPending → Valid At the choice of the pool, when processing a
:c:func:`PopFrame()`.
PopPending → Disabled At the choice of the pool, when processing a
:c:func:`PopFrame()`.
Disabled → Valid At the choice of the pool.
Disabled → Popframe Illegal.
======================= ====================================================
:mps:tag:`lw-frame.state-impl` Each AP contains 3 additional fields to hold this state::
mps_addr_t frameptr;
mps_bool_t enabled;
mps_bool_t lwPopPending;
:mps:tag:`lw-frame.enabled` The ``enabled`` slot holds the following values for
each state:
========== ==========
Valid :c:macro:`TRUE`
PopPending :c:macro:`TRUE`
Disabled :c:macro:`FALSE`
========== ==========
:mps:tag:`lw-frame.frameptr` The ``frameptr`` slot holds the following values
for each state:
========== ============================================
Valid :c:macro:`NULL`
PopPending Frame handle for most recently popped frame.
Disabled :c:macro:`NULL`
========== ============================================
:mps:tag:`lw-frame.lwPopPending` The ``lwPopPending`` slot holds the
following values for each state:
========== =========
Valid :c:macro:`FALSE`
PopPending :c:macro:`TRUE`
Disabled :c:macro:`FALSE`
========== =========
:mps:tag:`lw-frame.state-for-gc` It is not necessary for the tracer, format
code, pool, or any other part of the GC support in MPS to read either
of the two additional AP fields in order to scan a segment which
supports a lightweight allocation frame.
Synchronization
...............
:mps:tag:`lw-frame.sync` The purpose of the design is that mutator may
access the state of an AP without locking with MPS (via the external
functions). The design assumes the normal MPS restriction that an
operation on an AP may only be performed by a single mutator thread at
a time. Each of the operations on allocation frames counts as an
operation on an AP.
:mps:tag:`lw-frame.sync.pool` Pools are permitted to read or modify the
lightweight frame state of an AP only in response to an operation on
that AP.
:mps:tag:`lw-frame.sync.external` The external functions
:c:func:`mps_ap_frame_push()` and :c:func:`mps_ap_frame_pop()` are permitted to
read the values of the ``enabled`` and ``frameptr`` fields for the
supplied AP without claiming the arena lock. They are permitted to
modify the ``frameptr`` field if and only if ``enabled == FALSE``.
:mps:tag:`lw-frame.sync.trip` When a buffer trip happens, and the trap
wasn't set by MPS itself (that is, it wasn't because of a flip or for
logging), then the buffer code must check whether the AP has state
PopPending. If it does, the buffer code must call the Pool.
Implementation
..............
:mps:tag:`lw-frame.push` The external :c:func:`PushFrame()` operation
(:c:func:`mps_ap_frame_push()`) performs the following operations::
IF (!APIsTrapped(ap) && StateOfFrame(ap) == Valid && ap->init == ap->alloc)
*frame_o = ap->init;
ELSE
WITH_ARENA_LOCK
PerformInternalPushFrameOperation(...)
END
END
:mps:tag:`lw-frame.pop` The external :c:func:`PopFrame()` operation
(:c:func:`mps_ap_frame_pop()`) performs the following operations::
IF (StateOfFrame(ap) != Disabled)
TrapAP(ap); /* ensure next allocation or push involves the pool */
ap->frameptr = frame;
ap->lwpopPending = TRUE;
ELSE
WITH_ARENA_LOCK
PerformInternalPopFrameOperation(...)
END
END

533
mps/manual/html/_sources/design/arena.txt
View file

@ -1,6 +1,535 @@
.. _design-arena:
.. index::
pair: arena; design
.. _design-arena:
.. include:: ../../converted/arena.rst
Arena
=====
.. mps:prefix:: design.mps.arena
Introduction
------------
:mps:tag:`intro` This is the design of the arena structure.
:mps:tag:`readership` MPS developers.
Overview
--------
:mps:tag:`overview` The arena serves two purposes. It is a structure that is
the top-level state of the MPS, and as such contains a lot of fields
which are considered "global". And it provides raw memory to pools.
An arena belongs to a particular arena class. The class is selected
when the arena is created. Classes encapsulate both policy (such as
how pool placement preferences map into actual placement) and
mechanism (such as where the memory originates: operating system
virtual memory, client provided, or via malloc). Some behaviour
(mostly serving the "top-level datastructure" purpose) is implemented
by generic arena code, and some by arena class code.
Definitions
-----------
:mps:tag:`def.tract` Pools request memory from the arena by calling
:c:func:`ArenaAlloc()`. This returns a block comprising a contiguous sequence
of "tracts". A tract has a specific size (also known as the "arena
alignment", which typically corresponds to the operating system page
size) and all tracts are aligned to that size. "Tract" is also used
for the data structure used to manage tracts.
Requirements
------------
.. note::
Where do these come from? Need to identify and document the
sources of requirements so that they are traceable to client
requirements. Most of these come from the architectural design
(design.mps.architecture) or the fix function design
(design.mps.fix). Richard Brooksby, 1995-08-28.
They were copied from design.mps.arena.vm(1) and edited slightly.
David Jones, 1999-06-23.
Block management
................
:mps:tag:`req.fun.block.alloc` The arena must provide allocation of
contiguous blocks of memory.
:mps:tag:`req.fun.block.free` It must also provide freeing of contiguously
allocated blocks owned by a pool - whether or not the block was
allocated via a single request.
:mps:tag:`req.attr.block.size.min` The arena must support management of
blocks down to the size of the grain (page) provided by the virtual
mapping interface if a virtual memory interface is being used, or a
comparable size otherwise.
:mps:tag:`req.attr.block.size.max` It must also support management of blocks
up to the maximum size allowed by the combination of operating system
and architecture. This is derived from req.dylan.attr.obj.max (at
least).
:mps:tag:`req.attr.block.align.min` The alignment of blocks shall not be less
than :c:macro:`MPS_PF_ALIGN` for the architecture. This is so that pool
classes can conveniently guarantee pool allocated blocks are aligned
to :c:macro:`MPS_PF_ALIGN`. (A trivial requirement.)
:mps:tag:`req.attr.block.grain.max` The granularity of allocation shall not
be more than the grain size provided by the virtual mapping interface.
Address translation
...................
:mps:tag:`req.fun.trans` The arena must provide a translation from any
address to either an indication that the address is not in any tract
(if that is so) or the following data associated with the tract
containing that address:
:mps:tag:`req.fun.trans.pool` The pool that allocated the tract.
:mps:tag:`req.fun.trans.arbitrary` An arbitrary pointer value that the pool
can associate with the tract at any time.
:mps:tag:`req.fun.trans.white` The tracer whiteness information. That is, a
bit for each active trace that indicates whether this tract is white
(contains white objects). This is required so that the "fix" protocol
can run very quickly.
:mps:tag:`req.attr.trans.time` The translation shall take no more than @@@@
[something not very large -- drj 1999-06-23]
Iteration protocol
..................
:mps:tag:`req.iter` er, there's a tract iteration protocol which is
presumably required for some reason?
Arena partition
...............
:mps:tag:`req.fun.set` The arena must provide a method for approximating sets
of addresses.
:mps:tag:`req.fun.set.time` The determination of membership shall take no
more than @@@@ [something very small indeed]. (the non-obvious
solution is refsets)
Constraints
...........
:mps:tag:`req.attr.space.overhead` req.dylan.attr.space.struct implies that
the arena must limit the space overhead. The arena is not the only
part that introduces an overhead (pool classes being the next most
obvious), so multiple parts must cooperate in order to meet the
ultimate requirements.
:mps:tag:`req.attr.time.overhead` Time overhead constraint?
.. note::
How can there be a time "overhead" on a necessary component? David
Jones, 1999-06-23.
Architecture
------------
Statics
.......
:mps:tag:`static` There is no higher-level data structure than a arena, so in
order to support several arenas, we have to have some static data in
impl.c.arena. See impl.c.arena.static.
:mps:tag:`static.init` All the static data items are initialized when the
first arena is created.
:mps:tag:`static.serial` ``arenaSerial`` is a static :c:type:`Serial`, containing
the serial number of the next arena to be created. The serial of any
existing arena is less than this.
:mps:tag:`static.ring` ``arenaRing`` is the sentinel of the ring of arenas.
:mps:tag:`static.ring.init` ``arenaRingInit`` is a :c:type:`Bool` showing whether
the ring of arenas has been initialized.
:mps:tag:`static.ring.lock` The ring of arenas has to be locked when
traversing the ring, to prevent arenas being added or removed. This is
achieved by using the (non-recursive) global lock facility, provided
by the lock module.
:mps:tag:`static.check` The statics are checked each time any arena is
checked.
Arena classes
.............
.. c:type:: mps_arena_s *Arena
:mps:tag:`class` The :c:type:`Arena` data structure is designed to be subclassable
(see design.mps.protocol(0)). Clients can select what arena class
they'd like when instantiating one with :c:func:`mps_arena_create()`. The
arguments to :c:func:`mps_arena_create()` are class-dependent.
:mps:tag:`class.init` However, the generic :c:func:`ArenaInit()` is called from the
class-specific method, rather than vice versa, because the method is
responsible for allocating the memory for the arena descriptor and the
arena lock in the first place. Likewise, :c:func:`ArenaFinish()` is called
from the finish method.
:mps:tag:`class.fields` The ``alignment`` (for tract allocations) and
``zoneShift`` (for computing zone sizes and what zone an address is
in) fields in the arena are the responsibility of the each class, and
are initialized by the ``init`` method. The responsibility for
maintaining the ``commitLimit``, ``spareCommitted``, and
``spareCommitLimit`` fields is shared between the (generic) arena and
the arena class. ``commitLimit`` (see :mps:ref:`.commit-limit`) is changed by
the generic arena code, but arena classes are responsible for ensuring
the semantics. For ``spareCommitted`` and ``spareCommitLimit`` see
:mps:ref:`.spare-committed` below.
:mps:tag:`class.abstract` The basic arena class (:c:type:`AbstractArenaClass`) is
abstract and must not be instantiated. It provides little useful
behaviour, and exists primarily as the root of the tree of arena
classes. Each concrete class must specialize each of the class method
fields, with the exception of the describe method (which has a trivial
implementation) and the ``extend``, ``retract`` and
``spareCommitExceeded`` methods which have non-callable methods for
the benefit of arena classes which don't implement these features.
:mps:tag:`class.abstract.null` The abstract class does not provide dummy
implementations of those methods which must be overridden. Instead
each abstract method is initialized to :c:macro:`NULL`.
Tracts
......
:mps:tag:`tract` The arena allocation function :c:func:`ArenaAlloc()` allocates a
block of memory to pools, of a size which is aligned to the arena
alignment. Each alignment unit (grain) of allocation is represented by
a tract. Tracts are the hook on which the segment module is
implemented. Pools which don't use segments may use tracts for
associating their own data with each allocation grain.
:mps:tag:`tract.structure` The tract structure definition looks like this::
typedef struct TractStruct { /* Tract structure */
Pool pool; /* MUST BE FIRST (design.mps.arena.tract.field.pool) */
void *p; /* pointer for use of owning pool */
Addr base; /* Base address of the tract */
TraceSet white : TRACE_MAX; /* traces for which tract is white */
unsigned int hasSeg : 1; /* does tract have a seg in p? */
} TractStruct;
:mps:tag:`tract.field.pool` The pool field indicates to which pool the tract
has been allocated (:mps:ref:`.req.fun.trans.pool`). Tracts are only valid
when they are allocated to pools. When tracts are not allocated to
pools, arena classes are free to reuse tract objects in undefined
ways. A standard technique is for arena class implementations to
internally describe the objects as a union type of :c:type:`TractStruct` and
some private representation, and to set the pool field to :c:macro:`NULL`
when the tract is not allocated. The pool field must come first so
that the private representation can share a common prefix with
:c:type:`TractStruct`. This permits arena classes to determine from their
private representation whether such an object is allocated or not,
without requiring an extra field.
:mps:tag:`tract.field.p` The ``p`` field is used by pools to associate tracts
with other data (:mps:ref:`.req.fun.trans.arbitrary`). It's used by the
segment module to indicate which segment a tract belongs to. If a pool
doesn't use segments it may use the ``p`` field for its own purposes.
This field has the non-specific type ``(void *)`` so that pools can
use it for any purpose.
:mps:tag:`tract.field.hasSeg` The ``hasSeg`` bit-field is a Boolean which
indicates whether the ``p`` field is being used by the segment module.
If this field is :c:macro:`TRUE`, then the value of ``p`` is a :c:type:`Seg`.
``hasSeg`` is typed as an ``unsigned int``, rather than a :c:type:`Bool`.
This ensures that there won't be sign conversion problems when
converting the bit-field value.
:mps:tag:`tract.field.base` The base field contains the base address of the
memory represented by the tract.
:mps:tag:`tract.field.white` The white bit-field indicates for which traces
the tract is white (:mps:ref:`.req.fun.trans.white`). This information is also
stored in the segment, but is duplicated here for efficiency during a
call to ``TraceFix`` (see design.mps.trace.fix).
:mps:tag:`tract.limit` The limit of the tract's memory may be determined by
adding the arena alignment to the base address.
:mps:tag:`tract.iteration` Iteration over tracts is described in
design.mps.arena.tract-iter(0).
.. c:function:: Bool TractOfAddr(Tract *tractReturn, Arena arena, Addr addr)
:mps:tag:`tract.if.tractofaddr` The function :c:func:`TractOfAddr()` finds the tract
corresponding to an address in memory. (See :mps:ref:`.req.fun.trans`.)
If ``addr`` is an address which has been allocated to some pool, then
:c:func:`TractOfAddr()` returns :c:macro:`TRUE`, and sets ``*tractReturn`` to the
tract corresponding to that address. Otherwise, it returns :c:macro:`FALSE`.
This function is similar to :c:func:`TractOfBaseAddr()` (see
design.mps.arena.tract-iter.if.contig-base) but serves a more general
purpose and is less efficient.
:mps:tag:`tract.if.TRACT_OF_ADDR` :c:func:`TRACT_OF_ADDR()` is a macro version of
:c:func:`TractOfAddr()`. It's provided for efficiency during a call to
:c:func:`TraceFix()` (see design.mps.trace.fix.tractofaddr).
Control pool
............
:mps:tag:`pool` Each arena has a "control pool",
``arena->controlPoolStruct``, which is used for allocating MPS control
data structures by calling :c:func:`ControlAlloc()`.
Polling
.......
:mps:tag:`poll` :c:func:`ArenaPoll()` is called "often" by other code (for instance,
on buffer fill or allocation). It is the entry point for doing tracing
work. If the polling clock exceeds a set threshold, and we're not
already doing some tracing work (that is, ``insidePoll`` is not set),
it calls :c:func:`TracePoll()` on all busy traces.
:mps:tag:`poll.size` The actual clock is ``arena->fillMutatorSize``. This is
because internal allocation is only significant when copy segments are
being allocated, and we don't want to have the pause times to shrink
because of that. There is no current requirement for the trace rate to
guard against running out of memory.
.. note::
Clearly it really ought to: we have a requirement to not run out
of memory (see req.dylan.prot.fail-alloc, req.dylan.prot.consult),
and emergency tracing should not be our only story. David Jones,
1999-06-22.
:c:func:`BufferEmpty()` is not taken into account, because the splinter will
rarely be useable for allocation and we are wary of the clock running
backward.
:mps:tag:`poll.clamp` Polling is disabled when the arena is "clamped", in
which case ``arena->clamped`` is :c:macro:`TRUE`. Clamping the arena prevents
background tracing work, and further new garbage collections from
starting. Clamping and releasing are implemented by the :c:func:`ArenaClamp()`
and :c:func:`ArenaRelease()` methods.
:mps:tag:`poll.park` The arena is "parked" by clamping it, then polling until
there are no active traces. This finishes all the active collections
and prevents further collection. Parking is implemented by the
:c:func:`ArenaPark()` method.
Commit limit
............
:mps:tag:`commit-limit` The arena supports a client configurable "commit
limit" which is a limit on the total amount of committed memory. The
generic arena structure contains a field to hold the value of the
commit limit and the implementation provides two functions for
manipulating it: :c:func:`ArenaCommitLimit()` to read it, and
:c:func:`ArenaSetCommitLimit()` to set it. Actually abiding by the contract of
not committing more memory than the commit limit is left up to the
individual arena classes.
:mps:tag:`commit-limit.err` When allocation from the arena would otherwise
succeed but cause the MPS to use more committed memory than specified
by the commit limit :c:func:`ArenaAlloc()` should refuse the request and
return ``ResCOMMIT_LIMIT``.
:mps:tag:`commit-limit.err.multi` In the case where an :c:func:`ArenaAlloc()` request
cannot be fulfilled for more than one reason including exceeding the
commit limit then class implementations should strive to return a
result code other than ``ResCOMMIT_LIMIT``. That is,
``ResCOMMIT_LIMIT`` should only be returned if the *only* reason for
failing the :c:func:`ArenaAlloc()` request is that the commit limit would be
exceeded. The client documentation allows implementations to be
ambiguous with respect to which result code in returned in such a
situation however.
Spare committed (aka "hysteresis")
..................................
:mps:tag:`spare-committed` See :c:func:`mps_arena_spare_committed()`. The generic
arena structure contains two fields for the spare committed memory
fund: ``spareCommitted`` records the total number of spare committed
bytes; ``spareCommitLimit`` records the limit (set by the user) on the
amount of spare committed memory. ``spareCommitted`` is modified by
the arena class but its value is used by the generic arena code. There
are two uses: a getter function for this value is provided through the
MPS interface (:c:func:`mps_arena_spare_commit_limit_set()`), and by the
:c:func:`SetSpareCommitLimit()` function to determine whether the amount of
spare committed memory needs to be reduced. ``spareCommitLimit`` is
manipulated by generic arena code, however the associated semantics
are the responsibility of the class. It is the class's responsibility
to ensure that it doesn't use more spare committed bytes than the
value in ``spareCommitLimit``.
:mps:tag:`spare-commit-limit` The function :c:func:`ArenaSetSpareCommitLimit()` sets
the ``spareCommitLimit`` field. If the limit is set to a value lower
than the amount of spare committed memory (stored in
``spareCommitted``) then the class specific function
``spareCommitExceeded`` is called.
Locks
.....
:mps:tag:`lock.ring` :c:func:`ArenaAccess()` is called when we fault on a barrier.
The first thing it does is claim the non-recursive global lock to
protect the arena ring (see design.mps.lock(0)).
:mps:tag:`lock.arena` After the arena ring lock is claimed, :c:func:`ArenaEnter()` is
called on one or more arenas. This claims the lock for that arena.
When the correct arena is identified or we run out of arenas, the lock
on the ring is released.
:mps:tag:`lock.avoid` Deadlocking is avoided as described below:
:mps:tag:`lock.avoid.mps` Firstly we require the MPS not to fault (that is,
when any of these locks are held by a thread, that thread does not
fault).
:mps:tag:`lock.avoid.thread` Secondly, we require that in a multi-threaded
system, memory fault handlers do not suspend threads (although the
faulting thread will, of course, wait for the fault handler to
finish).
:mps:tag:`lock.avoid.conflict` Thirdly, we avoid conflicting deadlock between
the arena and global locks by ensuring we never claim the arena lock
when the recursive global lock is already held, and we never claim the
binary global lock when the arena lock is held.
Location dependencies
.....................
:mps:tag:`ld` Location dependencies use fields in the arena to maintain a
history of summaries of moved objects, and to keep a notion of time,
so that the staleness of location dependency can be determined.
Finalization
............
:mps:tag:`final` There is a pool which is optionally (and dynamically)
instantiated to implement finalization. The fields ``finalPool`` and
``isFinalPool`` are used.
Implementation
--------------
Tract cache
...........
:mps:tag:`tract.cache` When tracts are allocated to pools by :c:func:`ArenaAlloc()`,
the first tract of the block and it's base address are cached in arena
fields ``lastTract`` and ``lastTractBase``. The function
:c:func:`TractOfBaseAddr()` (see design.mps.arena.tract-iter.if.block-base(0))
checks against these cached values and only calls the class method on
a cache miss. This optimizes for the common case where a pool
allocates a block and then iterates over all its tracts (for example,
to attach them to a segment).
:mps:tag:`tract.uncache` When blocks of memory are freed by pools,
:c:func:`ArenaFree()` checks to see if the cached value for the most recently
allocated tract (see :mps:ref:`.tract.cache`) is being freed. If so, the cache
is invalid, and must be reset. The ``lastTract`` and ``lastTractBase``
fields are set to :c:macro:`NULL`.
Control pool
............
:mps:tag:`pool.init` The control pool is initialized by a call to
:c:func:`PoolInit()` during :c:func:`ArenaCreate()`.
:mps:tag:`pool.ready` All the other fields in the arena are made checkable
before calling :c:func:`PoolInit()`, so :c:func:`PoolInit()` can call
``ArenaCheck(arena)``. The pool itself is, of course, not checkable,
so we have a field ``arena->poolReady``, which is false until after
the return from :c:func:`PoolInit()`. :c:func:`ArenaCheck()` only checks the pool if
``poolReady``.
Traces
......
:mps:tag:`trace` ``arena->trace[ti]`` is valid if and only if
``TraceSetIsMember(arena->busyTraces, ti)``.
:mps:tag:`trace.create` Since the arena created by :c:func:`ArenaCreate()` has
``arena->busyTraces = TraceSetEMPTY``, none of the traces are
meaningful.
:mps:tag:`trace.invalid` Invalid traces have signature ``SigInvalid``, which
can be checked.
Polling
.......
:mps:tag:`poll.fields` There are three fields of a arena used for polling:
``pollThreshold``, ``insidePoll``, and ``clamped`` (see above).
``pollThreshold`` is the threshold for the next poll: it is set at the
end of :c:func:`ArenaPoll()` to the current polling time plus
:c:macro:`ARENA_POLL_MAX`.
Location dependencies
.....................
:mps:tag:`ld.epoch` ``arena->epoch`` is the "current epoch". This is the
number of 'flips' of traces in the arena since the arena was created.
From the mutator's point of view locations change atomically at flip.
:mps:tag:`ld.history` ``arena->history`` is an array of :c:macro:`ARENA_LD_LENGTH`
elements of type ``RefSet``. These are the summaries of moved objects
since the last :c:macro:`ARENA_LD_LENGTH` epochs. If ``e`` is one of these
recent epochs, then ::
arena->history[e % ARENA_LD_LENGTH]
is a summary of (the original locations of) objects moved since epoch
``e``.
:mps:tag:`ld.prehistory` ``arena->prehistory`` is a ``RefSet`` summarizing
the original locations of all objects ever moved. When considering
whether a really old location dependency is stale, it is compared with
this summary.
Roots
.....
:mps:tag:`root-ring` The arena holds a member of a ring of roots in the
arena. It holds an incremental serial which is the serial of the next
root.

206
mps/manual/html/_sources/design/arenavm.txt Normal file
View file

@ -0,0 +1,206 @@
.. _design-arenavm:
.. index::
pair: virtual memory arena; design
pair: VM arena; design
Virtual Memory Arena
====================
.. mps:prefix:: design.mps.arena.vm
pair: virtual memory arena; design
pair: VM arena; design
Introduction
------------
:mps:tag:`intro` This document describes the detailed design of the Virtual
Memory Arena Class of the Memory Pool System. The VM Arena Class is
just one class available in the MPS. The generic arena part is
described in design.mps.arena.
Overview
--------
:mps:tag:`overview` VM arenas provide blocks of memory to all other parts of
the MPS in the form of "tracts" using the virtual mapping interface
(design.mps.vm) to the operating system. The VM Arena Class is not
expected to be provided on platforms that do not have virtual memory
(like MacOS, os.s7(1)).
:mps:tag:`overview.gc` The VM Arena Class provides some special services on
these blocks in order to facilitate garbage collection:
:mps:tag:`overview.gc.zone` Allocation of blocks with specific zones. This
means that the generic fix function (design.mps.fix) can use a fast
refset test to eliminate references to addresses that are not in the
condemned set. This assumes that a pool class that uses this placement
appropriately is being used (such as the generation placement policy
used by AMC: see design.mps.poolamc(1)) and that the pool selects the
condemned sets to coincide with zone stripes.
:mps:tag:`overview.gc.tract` A fast translation from addresses to tract.
(See design.mps.arena.req.fun.trans)
Notes
-----
:mps:tag:`note.refset` Some of this document simply assumes that RefSets
(see the horribly incomplete design.mps.refset) have been chosen as
the solution for design.mps.arena.req.fun.set. It's a lot simpler that
way. Both to write and understand.
Requirements
------------
Most of the requirements are in fact on the generic arena (see
design.mps.arena.req). However, many of those requirements can only be
met by a suitable arena class design.
Requirements particular to this arena class:
Placement
.........
:mps:tag:`req.fun.place` It must be possible for pools to obtain tracts at
particular addresses. Such addresses shall be declared by the pool
specifying what refset zones the tracts should lie in and what refset
zones the tracts should not lie in. It is acceptable for the arena to
not always honour the request in terms of placement if it has run out
of suitable addresses.
Arena partition
...............
:mps:tag:`req.fun.set` See design.mps.arena.req.fun.set. The approximation
to sets of address must cooperate with the placement mechanism in the
way required by :mps:ref:`.req.fun.place` (above).
Architecture
------------
:mps:tag:`arch.memory` The underlying memory is obtained from whatever
Virtual Memory interface (see design.mps.vm). @@@@ Explain why this is
used.
Solution ideas
--------------
:mps:tag:`idea.grain` Set the arena granularity to the grain provided by the
virtual mapping module.
:mps:tag:`idea.mem` Get a single large contiguous address area from the
virtual mapping interface and divide that up.
:mps:tag:`idea.table` Maintain a table with one entry per grain in order to
provide fast mapping (shift and add) between addresses and table
entries.
:mps:tag:`idea.table.figure` [missing figure]
:mps:tag:`idea.map` Store the pointers (:mps:ref:`.req.fun.trans`) in the table
directly for every grain.
:mps:tag:`idea.zones` Partition the managed address space into zones (see
idea.zones) and provide the set approximation as a reference
signature.
:mps:tag:`idea.first-fit` Use a simple first-fit allocation policy for
tracts within each zone (:mps:ref:`.idea.zones`). Store the freelist in the
table (:mps:ref:`.idea.table`).
:mps:tag:`idea.base` Store information about each contiguous area (allocated
of free) in the table entry (:mps:ref:`.idea.table`) corresponding to the base
address of the area.
:mps:tag:`idea.shadow` Use the table (:mps:ref:`.idea.table`) as a "shadow" of the
operating system's page table. Keep information such as last access,
protection, etc. in this table, since we can't get at this information
otherwise.
:mps:tag:`idea.barrier` Use the table (:mps:ref:`.idea.table`) to implement the
software barrier. Each segment can have a read and/or write barrier
placed on it by each process. (:mps:tag:`idea.barrier.bits` Store a
bit-pattern which remembers which process protected what.) This will
give a fast translation from a barrier-protected address to the
barrier handler via the process table.
:mps:tag:`idea.demand-table` For a 1 GiB managed address space with a 4 KiB
page size, the table will have 256K-entries. At, say, four words per
entry, this is 4 MiB of table. Although this is only an 0.4%, the
table shouldn't be preallocated or initially it is an infinite
overhead, and with 1 MiB active, it is a 300% overhead! The address
space for the table should be reserved, but the pages for it mapped
and unmapped on demand. By storing the table in a tract, the status of
the table's pages can be determined by looking at it's own entries in
itself, and thus the translation lookup (:mps:ref:`.req.fun.trans`) is slowed
to two lookups rather than one.
:mps:tag:`idea.pool` Make the Arena Manager a pool class. Arena
initialization becomes pool creation. Tract allocation becomes
:c:func:`PoolAlloc()`. Other operations become class-specific operations on
the "arena pool".
Data structures
---------------
:mps:tag:`tables` There are two table data structures: a page table, and an
alloc table.
:mps:tag:`table.page.map` Each page in the VM has a corresponding page table
entry.
:mps:tag:`table.page.linear` The table is a linear array of PageStruct
entries; there is a simple mapping between the index in the table and
the base address in the VM. Namely:
- index to base address: ``base-address = arena-base + (index * page-size)``
- base address to index: ``index = (base-address - arena-base) / page-size``
:mps:tag:`table.page.partial` The table is partially mapped on an
"as-needed" basis. The function :c:func:`unusedTablePages()` identifies
entirely unused pages occupied by the page table itself (that is,
those pages of the page table which are occupied by :c:type:`PageStruct`
objects which all describe free pages). Tract allocation and freeing
use this function to map and unmap the page table with no hysteresis.
(There is restriction on the parameters you may pass to
:c:func:`unusedTablePages()`.)
:mps:tag:`table.page.tract` Each page table entry contains a tract, which is
only valid if it is allocated to a pool. If it is not allocated to a
pool, the fields of the tract are used for other purposes. (See
design.mps.arena.tract.field.pool)
:mps:tag:`table.alloc` The alloc table is a simple bit table (implemented
using the BT module, design.mps.bt).
:mps:tag:`table.alloc.map` Each page in the VM has a corresponding alloc
table entry.
:mps:tag:`table.alloc.semantics` The bit in the alloc table is set iff the
corresponding page is allocated (to a pool).
Notes
-----
:mps:tag:`fig.page` How the pages in the arena area are represented in the
tables.
[missing figure]
:mps:tag:`fig.count` How a count table can be used to partially map the page
table, as proposed in request.dylan.170049.sol.map.
[missing figure]

707
mps/manual/html/_sources/design/bt.txt
View file

@ -1,6 +1,709 @@
.. _design-bt:
.. index::
pair: bit tables; design
.. _design-bt:
.. include:: ../../converted/bt.rst
Bit tables
==========
.. mps:prefix:: design.mps.bt
Introduction
------------
:mps:tag:`intro` This is the design of the Bit Tables module. A Bit Table is a
linear array of bits. A Bit Table of length *n* is indexed using an
integer from 0 up to (but not including) *n*. Each bit in a Bit Table
can hold either the value 0 (aka :c:macro:`FALSE`) or 1 (aka :c:macro:`TRUE`). A
variety of operations are provided including: set, reset, and
retrieve, individual bits; set and reset a contiguous range of bits;
search for a contiguous range of reset bits; making a "negative image"
copy of a range.
:mps:tag:`readership` MPS developers.
Definitions
-----------
:mps:tag:`def.set` **Set**
Used as a verb meaning to assign the value 1 or :c:macro:`TRUE` to a bit.
Used descriptively to denote a bit containing the value 1. Note 1
and :c:macro:`TRUE` are synonyms in MPS C code (see
design.mps.type(0).bool.value).
:mps:tag:`def.reset` **Reset**
Used as a verb meaning to assign the value 0 or :c:macro:`FALSE` to a
bit. Used descriptively to denote a bit containing the value 0.
Note 0 and :c:macro:`FALSE` are synonyms in MPS C code (see
design.mps.type(0).bool.value).
.. note::
Consider using "fill/empty" or "mark/clear" instead of
"set/reset", set/reset is probably a hangover from drj's z80
hacking days -- drj 1999-04-26
:mps:tag:`def.bt` **Bit Table**
A Bit Table is a mapping from [0, *n*) to {0,1} for some *n*,
represented as a linear array of bits.
_`..def.bt.justify`: They are called *bit tables* because a single
bit is used to encode whether the image of a particular integer
under the map is 0 or 1.
:mps:tag:`def.range` **Range**
A contiguous sequence of bits in a Bit Table. Ranges are typically
specified as a *base*--*limit* pair where the range includes the
position specified by the base, but excludes that specified by the
limit. The mathematical interval notation for half-open intervals,
[*base*, *limit*), is used.
Requirements
------------
:mps:tag:`req.bit` The storage for a Bit Table of *n* bits shall take no more
than a small constant addition to the storage required for *n* bits.
_`..req.bit.why`: This is so that clients can make some predictions
about how much storage their algorithms use. A small constant is
allowed over the minimal for two reasons: inevitable implementation
overheads (such as only being able to allocate storage in multiples of
32 bits), extra storage for robustness or speed (such as signature and
length fields).
:mps:tag:`req.create` A means to create Bit Tables. :mps:tag:`req.create.why` Obvious.
:mps:tag:`req.destroy` A means to destroy Bit Tables. _`.req.destroy.why`
Obvious.
:mps:tag:`req.ops` The following operations shall be supported:
* :mps:tag:`req.ops.get` **Get**. Get the value of a bit at a specified
index.
* :mps:tag:`req.ops.set` **Set**. Set a bit at a specified index.
* :mps:tag:`req.ops.reset` **Reset**. Reset a bit at a specified index.
:mps:tag:`req.ops.minimal.why` Get, Set, Reset, are the minimal operations.
All possible mappings can be created and inspected using these
operations.
* :mps:tag:`req.ops.set.range` **SetRange**. Set a range of bits.
:mps:tag:`req.ops.set.range.why` It's expected that clients will often want
to set a range of bits; providing this operation allows the
implementation of the BT module to make the operation efficient.
* :mps:tag:`req.ops.reset.range` **ResetRange**. Reset a range of
bits. :mps:tag:`req.ops.reset.range.why` as for SetRange, see
:mps:ref:`.req.ops.set.range.why`.
* :mps:tag:`req.ops.test.range.set` **IsSetRange**. Test whether a range
of bits are all set. :mps:tag:`req.ops.test.range.set.why` Mostly
for checking. For example, often clients will know that a range they
are about to reset is currently all set, they can use this operation
to assert that fact.
* :mps:tag:`req.ops.test.range.reset` **IsResetRange**. Test whether a
range of bits are all reset. _`.req.ops.test.range.reset.why`
As for IsSetRange, see :mps:ref:`.req.ops.test.range.set.why`.
* :mps:tag:`req.ops.find` Find a range (which we'll denote [*i*, *j*)) of at
least *L* reset bits that lies in a specified subrange of the entire
Bit Table. Various find operations are required according to the
(additional) properties of the required range:
* :mps:tag:`req.ops.find.short.low` **FindShortResetRange**. Of all
candidate ranges, find the range with least *j* (find the leftmost
range that has at least *L* reset bits and return just enough of
that). :mps:tag:`req.ops.find.short.low.why` Required by client and VM
arenas to allocate segments. The arenas implement definite
placement policies (such as lowest addressed segment first) so
they need the lowest (or highest) range that will do. It's not
currently useful to allocate segments larger than the requested
size, so finding a short range is sufficient.
* :mps:tag:`req.ops.find.short.high` **FindShortResetRangeHigh**. Of all
candidate ranges, find the range with greatest *i* (find the
rightmost range that has at least *L* reset bits and return just
enough of that). :mps:tag:`req.ops.find.short.high.why` Required by arenas
to implement a specific segment placement policy (highest
addressed segment first).
* :mps:tag:`req.ops.find.long.low` **FindLongResetRange**. Of all candidate
ranges, identify the ranges with least *i* and of those find the
one with greatest *j* (find the leftmost range that has at least
*L* reset bits and return all of it). _`.req.ops.find.long.low.why`
Required by the mark and sweep Pool Classes (AMS, AWL, LO) for
allocating objects (filling a buffer). It's more efficient to fill
a buffer with as much memory as is conveniently possible. There's
no strong reason to find the lowest range but it's bound to have
some beneficial (small) cache effect and makes the algorithm more
predictable.
* :mps:tag:`req.ops.find.long.high` **FindLongResetRangeHigh**. Provided,
but not required, see :mps:ref:`.non-req.ops.find.long.high`.
* :mps:tag:`req.ops.copy` Copy a range of bits from one Bit Table to another
Bit Table. Various copy operations are required:
* :mps:tag:`req.ops.copy.simple` Copy a range of bits from one Bit Table to
the same position in another Bit Table.
:mps:tag:`req.ops.copy.simple.why` Required to support copying of the
tables for the "low" segment during segment merging and splitting,
for pools using tables (for example, ``PoolClassAMS``).
* :mps:tag:`req.ops.copy.offset` Copy a range of bits from one Bit Table to
an offset position in another Bit Table.
:mps:tag:`req.ops.copy.offset.why` Required to support copying of the
tables for the "high" segment during segment merging and
splitting, for pools which support this (currently none, as of
2000-01-17).
* :mps:tag:`req.ops.copy.invert` Copy a range of bits from one Bit Table to
the same position in another Bit Table inverting all the bits in
the target copy. :mps:tag:`req.ops.copy.invert.why` Required by colour
manipulation code in ``PoolClassAMS`` and ``PoolClassLO``.
:mps:tag:`req.speed` Operations shall take no more than a few memory
operations per bit manipulated. :mps:tag:`req.speed.why` Any slower
would be gratuitous.
:mps:tag:`req.speed.fast` The following operations shall be very fast:
* :mps:tag:`req.speed.fast.find.short` FindShortResRange (the
operation used to meet :mps:ref:`.req.ops.find.short.low`)
FindShortResRangeHigh (the operation used to meet
:mps:ref:`.req.ops.find.short.high`).
:mps:tag:`req.speed.fast.find.short.why` These two are used by the client
arena (design.mps.arena.client) and the VM arena
(design.mps.arena.vm) for finding segments in page tables. The
operation will be used sufficiently often that its speed will
noticeably affect the overall speed of the MPS. They will be called
with a length equal to the number of pages in a segment. Typical
values of this length depend on the pool classes used and their
configuration, but we can expect length to be small (1 to 16)
usually. We can expect the Bit Table to be populated densely where
it is populated at all, that is set bits will tend to be clustered
together in subranges.
* :mps:tag:`req.speed.fast.find.long` FindLongResRange (the operation
used to meet :mps:ref:`.req.ops.find.long.low`)
:mps:tag:`req.speed.fast.find.long.why` Used in the allocator for
``PoolClassAWL`` (design.mps.poolawl(1)), ``PoolClassAMS``
(design.mps.poolams(2)), ``PoolClassEPVM`` (design.mps.poolepvm(0)).
Of these AWL and EPVM have speed requirements. For AWL the length of
range to be found will be the length of a Dylan table in words.
According to mail.tony.1999-05-05.11-36(0), only <entry-vector>
objects are allocated in AWL (though not all <entry-vector> objects
are allocated in AWL), and the mean length of an <entry-vector>
object is 486 Words. No data for EPVM alas.
:mps:tag:`req.speed.fast.other.why` We might expect mark and sweep pools to
make use of Bit Tables, the MPS has general requirements to support
efficient mark and sweep pools, so that imposes general speed
requirements on Bit Tables.
Non requirements
----------------
The following are not requirements but the current design could
support them with little modification or does support them. Often they
used to be requirements, but are no longer, or were added
speculatively or experimentally but aren't currently used.
* :mps:tag:`non-req.ops.test.range.same` **RangesSame**. Test whether two
ranges that occupy the same positions in different Bit Tables are
the same. This used to be required by ``PoolClassAMS``, but is no
longer. Currently (1999-05-04) the functionality still exists.
* :mps:tag:`non-req.ops.find.long.high` **FindLongResetRangeHigh**. (see
:mps:ref:`.req.ops.find`) Of all candidate ranges, identify the ranges with
greatest *j* and of those find the one with least *i* (find the
rightmost range that has at least *L* reset bits and return all of
it). Provided for symmetry but only currently used by the BT tests
and ``cbstest.c``.
Background
----------
:mps:tag:`background` Originally Bit Tables were used and implemented
by ``PoolClassLO`` (design.mps.poollo). It was
decided to lift them out into a separate module when designing the
Pool to manage Dylan Weak Tables which is also a mark and sweep pool
and will make use of Bit Tables (see design.mps.poolawl).
:mps:tag:`background.analysis` analysis.mps.bt(0) contains
some of the analysis of the design decisions that were and were not
made in this document.
Clients
-------
:mps:tag:`clients` Bit Tables are used throughout the MPS but the important
uses are in the client and VM arenas (design.mps.arena.client(0) and
design.mps.arena.vm(1)) a bit table is used to record whether each
page is free or not; several pool classes (``PoolClassLO``,
``PoolClassEPVM``, ``PoolClassAMS``) use bit tables to record which
locations are free and also to store colour.
Overview
--------
:mps:tag:`over` Mostly, the design is as simple as possible. The significant
complications are iteration (see :mps:ref:`.iteration` below) and searching
(see :mps:ref:`.fun.find-res-range` below) because both of these are required
to be fast.
Interface
---------
.. c:type:: Word *BT
:mps:tag:`if.representation.abstract` A Bit Table is represented by the type
:c:type:`BT`.
:mps:tag:`if.declare` The module declares a type :c:type:`BT` and a prototype for
each of the functions below. The type is declared in impl.h.mpmtypes,
the prototypes are declared in impl.h.mpm. Some of the functions are
in fact implemented as macros in the usual way
(doc.mps.ref-man.if-conv(0).macro.std).
:mps:tag:`if.general.index` Many of the functions specified below take
indexes. If otherwise unspecified an index must be in the interval [0,
*n*) (note, up to, but not including, *n*) where *n* is the number of
bits in the relevant Bit Table (as passed to the :c:func:`BTCreate()`
function).
:mps:tag:`if.general.range` Where a range is specified by two indexes (*base*
and *limit*), the index *base*, which specifies the beginning of the
range, must be in the interval [0, *n*), and the index *limit*, which
specifies the end of the range, must be in the interval [1, *n*] (note
can be *n*), and *base* must be strictly less than *limit* (empty
ranges are not allowed). Sometimes *i* and *j* are used instead of
*base* and *limit*.
.. c:function:: Res BTCreate(BT *btReturn, Arena arena, Count n)
:mps:tag:`if.create` Attempts to create a table of length ``n`` in the arena
control pool, putting the table in ``*btReturn``. Returns ``ResOK`` if
and only if the table is created OK. The initial values of the bits in
the table are undefined (so the client should probably call
:c:func:`BTResRange()` on the entire range before using the :c:type:`BT`). Meets
:mps:ref:`.req.create`.
.. c:function:: void BTDestroy(BT t, Arena arena, Count n)
:mps:tag:`if.destroy` Destroys the table ``t``, which must have been created
with :c:func:`BTCreate()`. The value of argument ``n`` must be same as the
value of the argument passed to :c:func:`BTCreate()`. Meets
:mps:ref:`.req.destroy`.
.. c:function:: size_t BTSize(Count n)
:mps:tag:`if.size` ``BTSize(n)`` returns the number of bytes needed for a Bit
Table of ``n`` bits. :c:func:`BTSize()` is a macro, but ``(BTSize)(n)`` will
assert if ``n`` exceeds ``COUNT_MAX - MPS_WORD_WIDTH + 1``. This is
used by clients that allocate storage for the :c:type:`BT` themselves.
Before :c:func:`BTCreate()` and :c:func:`BTDestroy()` were implemented that was the
only way to allocate a Bit Table, but is now deprecated.
.. c:function:: int BTGet(BT t, Index i)
:mps:tag:`if.get` ``BTGet(t, i)`` returns the ``i``-th bit of the table ``t``
(that is, the image of ``i`` under the mapping). Meets
:mps:ref:`.req.ops.get`.
.. c:function:: void BTSet(BT t, Index i)
:mps:tag:`if.set` ``BTSet(t, i)`` sets the ``i``-th bit of the table ``t`` (to
1). ``BTGet(t, i)`` will now return 1. Meets :mps:ref:`.req.ops.set`.
.. c:function:: void BTRes(BT t, Index i)
:mps:tag:`if.res` ``BTRes(t, i)`` resets the ``i``-th bit of the table ``t``
(to 0). ``BTGet(t, i)`` will now return 0. Meets :mps:ref:`.req.ops.reset`.
.. c:function:: void BTSetRange(BT t, Index base, Index limit)
:mps:tag:`if.set-range` ``BTSetRange(t, base, limit)`` sets the range of bits
[``base``, ``limit``) in the table ``t``. ``BTGet(t, x)`` will now
return 1 for ``base`` ≤ ``x`` < ``limit``. Meets
:mps:ref:`.req.ops.test.range.set`.
.. c:function:: void BTResRange(BT t, Index base, Index limit)
:mps:tag:`if.res-range` ``BTResRange(t, base, limit)`` resets the range of
bits [``base``, ``limit``) in the table ``t``. ``BTGet(t, x)`` will
now return 0 for ``base`` ≤ ``x`` < ``limit``. Meets
:mps:ref:`.req.ops.test.range.reset`.
.. c:function:: Bool BTIsSetRange(BT bt, Index base, Index limit)
:mps:tag:`if.test.range.set` Returns :c:macro:`TRUE` if all the bits in the range
[``base``, ``limit``) are set, :c:macro:`FALSE` otherwise. Meets
:mps:ref:`.req.ops.test.range.set`.
.. c:function:: Bool BTIsResRange(BT bt, Index base, Index limit)
:mps:tag:`if.test.range.reset` Returns :c:macro:`TRUE` if all the bits in the range
[``base``, ``limit``) are reset, :c:macro:`FALSE` otherwise. Meets
:mps:ref:`.req.ops.test.range.reset`.
``Bool BTRangesSame(BT BTx, BT BTy, Index base, Index limit);``
:mps:tag:`if.test.range.same` returns :c:macro:`TRUE` if ``BTGet(BTx,i)`` equals
``BTGet(BTy,i)`` for ``i`` in [``base``, ``limit``), and :c:macro:`FALSE`
otherwise. Meets :mps:ref:`.non-req.ops.test.range.same`.
:mps:tag:`if.find.general` There are four functions (below) to find reset
ranges. All the functions have the same prototype (for symmetry)::
Bool find(Index *baseReturn, Index *limitReturn,
BT bt,
Index searchBase, Index searchLimit,
Count length);
where ``bt`` is the Bit Table in which to search. ``searchBase`` and
``searchLimit`` specify a subset of the Bit Table to use, the
functions will only find ranges that are subsets of [``searchBase``,
``searchLimit``) (when set, ``*baseReturn`` will never be less than
``searchBase`` and ``*limitReturn`` will never be greater than
``searchLimit``). ``searchBase`` and ``searchLimit`` specify a range
that must conform to the general range requirements for a range [*i*,
*j*), as per :mps:ref:`.if.general.range` modified appropriately. ``length``
is the number of contiguous reset bits to find; it must not be bigger
than ``searchLimit - searchBase`` (that would be silly). If a suitable
range cannot be found the function returns :c:macro:`FALSE` (0) and leaves
``*baseReturn`` and ``*limitReturn`` untouched. If a suitable range is
found then the function returns the range's base in ``*baseReturn``
and its limit in ``*limitReturn`` and returns :c:macro:`TRUE` (1).
.. c:function:: Bool BTFindShortResRange(Index *baseReturn, Index *limitReturn, BT bt, Index searchBase, Index searchLimit, Count length)
:mps:tag:`if.find-short-res-range` Finds a range of reset bits in the table,
starting at ``searchBase`` and working upwards. This function is
intended to meet :mps:ref:`.req.ops.find.short.low` so it will find the
leftmost range that will do, and never finds a range longer than the
requested length (the intention is that it will not waste time
looking).
.. c:function:: Bool BTFindShortResRangeHigh(Index *baseReturn, Index *limitReturn, BT bt, Index searchBase, Index searchLimit, Count length)
:mps:tag:`if.find-short-res-range-high` Finds a range of reset bits in the
table, starting at ``searchLimit`` and working downwards. This
function is intended to meet :mps:ref:`.req.ops.find.short.high` so it will
find the rightmost range that will do, and never finds a range longer
than the requested length.
.. c:function:: Bool BTFindLongResRange(Index *baseReturn, Index *limitReturn, BT bt, Index searchBase, Index searchLimit, Count length)
:mps:tag:`if.find-long-res-range` Finds a range of reset bits in the table,
starting at ``searchBase`` and working upwards. This function is
intended to meet :mps:ref:`.req.ops.find.long.low` so it will find the
leftmost range that will do and returns all of that range (which can
be longer than the requested length).
.. c:function:: Bool BTFindLongResRangeHigh(Index *baseReturn, Index *limitReturn, BT bt, Index searchBase, Index searchLimit, Count length)
:mps:tag:`if.find-long-res-range-high` Finds a range of reset bits in the
table, starting at ``searchLimit`` and working downwards. This
function is intended to meet :mps:ref:`.req.ops.find.long.high` so it will
find the rightmost range that will do and returns all that range
(which can be longer than the requested length).
.. c:function:: void BTCopyRange(BT fromBT, BT toBT, Index base, Index limit)
:mps:tag:`if.copy-range` Overwrites the ``i``-th bit of ``toBT`` with the
``i``-th bit of ``fromBT``, for all ``i`` in [``base``, ``limit``).
Meets :mps:ref:`.req.ops.copy.simple`.
.. c:function:: void BTCopyOffsetRange(BT fromBT, BT toBT, Index fromBase, Index fromLimit, Index toBase, Index toLimit)
:mps:tag:`if.copy-offset-range` Overwrites the ``i``-th bit of ``toBT`` with
the ``j``-th bit of ``fromBT``, for all ``i`` in [``toBase``,
``toLimit``) and corresponding ``j`` in [``fromBase``, ``fromLimit``).
Each of these ranges must be the same size. This might be
significantly less efficient than :c:func:`BTCopyRange()`. Meets
:mps:ref:`.req.ops.copy.offset`.
.. c:function:: void BTCopyInvertRange(BT fromBT, BT toBT, Index base, Index limit)
:mps:tag:`if.copy-invert-range` Overwrites the ``i``-th bit of ``toBT`` with
the inverse of the ``i``-th bit of ``fromBT``, for all ``i`` in
[``base``, ``limit``). Meets :mps:ref:`.req.ops.copy.invert`.
Detailed design
---------------
Data structures
...............
:mps:tag:`datastructure` Bit Tables will be represented as (a pointer to) an
array of :c:type:`Word`. A plain array is used instead of the more usual
design convention of implementing an abstract data type as a structure
with a signature (see guide.impl.c.adt(0)).
:mps:tag:`datastructure.words.justify` The type :c:type:`Word` is used as it will
probably map to the object that can be most efficiently accessed on
any particular platform. :mps:tag:`datastructure.non-adt.justify` The usual
abstract data type convention was not followed because (i) The initial
design (drj) was lazy, (ii) Bit Tables are more likely to come in
convenient powers of two with the extra one or two words overhead.
However, the loss of checking is severe. Perhaps it would be better to
use the usual abstract data type style.
Functions
.........
:mps:tag:`fun.size` :c:func:`BTSize()`. Since a Bit Table is an array of :c:type:`Word`, the
size of a Bit Table of *n* bits is simply the number of words that it
takes to store *n* bits times the number of bytes in a word. This is
``ceiling(n/MPS_WORD_WIDTH)*sizeof(Word).`` :mps:tag:`fun.size.justify` Since
there can be at most ``MPS_WORD_WIDTH - 1`` unused bits in the entire
table, this satisfies :mps:ref:`.req.bit`.
:mps:tag:`index` The designs for the following functions use a decomposition
of a bit-index, ``i``, into two parts, ``iw``, ``ib``.
* :mps:tag:`index.word` ``iw`` is the "word-index" which is the index into the
word array of the word that contains the bit referred to by the
bit-index. ``iw = i / MPS_WORD_WIDTH``. Since :c:macro:`MPS_WORD_WIDTH` is
a power of two, this is the same as ``iw = i >> MPS_WORD_SHIFT``.
The latter expression is used in the code. :mps:tag:`index.word.justify` The
compiler is more likely to generate good code without the divide.
* :mps:tag:`index.sub-word` ``ib`` is the "sub-word-index" which is the index
of the bit referred to by the bit-index in the above word. ``ib = i
% MPS_WORD_WIDTH``. Since :c:macro:`MPS_WORD_WIDTH` is a power of two, this
is the same as ``ib = i & ~((Word)-1<<MPS_WORD_SHIFT)``. The latter
expression is used in the code. :mps:tag:`index.sub-word.justify` The
compiler is more likely to generate good code without the modulus.
:mps:tag:`index.justify.dubious` The above justifications are dubious; gcc
2.7.2 (with -O2) running on a sparc (zaphod) produces identical code
for the following two functions::
unsigned long f(unsigned long i) {
return i/32 + i%32;
}
unsigned long g(unsigned long i) {
return (i>>5) + (i&31);
}
.. c:function:: ACT_ON_RANGE(Index base, Index limit, single_action, bits_action, word_action)
.. c:function:: ACT_ON_RANGE_HIGH(Index base, Index limit, single_action, bits_action, word_action)
:mps:tag:`iteration` Many of the following functions involve iteration over
ranges in a Bit Table. This is performed on whole words rather than
individual bits, whenever possible (to improve speed). This is
implemented internally by the macros :c:func:`ACT_ON_RANGE()` and
:c:func:`ACT_ON_RANGE_HIGH()` for iterating over the range forwards and
backwards respectively. These macros do not form part of the interface
of the module, but are used extensively in the implementation. The
macros are often used even when speed is not an issue because it
simplifies the implementation and makes it more uniform. The iteration
macros take the parameters ``base``, ``limit``, ``single_action``,
``bits_action``, and ``word_action``:
* ``base`` and ``limit`` are of type :c:type:`Index` and define the range of
the iteration.
* ``single_action`` is the name of a macro which will be used for
iterating over bits in the table individually. This macro must take
a single :c:type:`Index` parameter corresponding to the index for the bit.
The expansion of the macro must not contain ``break`` or
``continue`` because it will be called from within a loop from the
expansion of :c:func:`ACT_ON_RANGE()`.
* ``bits_action`` is the name of a macro which will be used for
iterating over part-words. This macro must take parameters
``wordIndex``, ``base``, ``limit`` where ``wordIndex`` is the index
into the array of words, and ``base`` and ``limit`` define a range
of bits within the indexed word.
* ``word_action`` is the name of a macro which will be used for
iterating over whole-words. This macro must take the single
parameter ``wordIndex`` which is the index of the whole-word in the
array. The expansion of the macro must not contain ``break`` or
``continue`` because it will be called from within a loop from the
expansion of :c:func:`ACT_ON_RANGE()`.
:mps:tag:`iteration.exit` The expansion of the ``single_action``,
``bits_action``, and ``word_action`` macros is allowed to contain
``return`` or ``goto`` to terminate the iteration early. This is used
by the test (:mps:ref:`.fun.test.range.set`) and find (:mps:ref:`.fun.find`)
operations.
:mps:tag:`iteration.small` If the range is sufficiently small only the
``single_action`` macro will be used, as this is more efficient in
practice. The choice of what constitutes a small range is made
entirely on the basis of experimental performance results (and
currently, 1999-04-27, a "small range" is 6 bits or fewer. See
change.mps.epcore.brisling.160181 for some justification). Otherwise
(for a bigger range) ``bits_action`` is used on the part words at
either end of the range (or the whole of the range it if it fits in a
single word), and ``word_action`` is used on the words that comprise
the inner portion of the range.
The implementation of :c:func:`ACT_ON_RANGE()` (and :c:func:`ACT_ON_RANGE_HIGH()`) is
simple enough. It decides which macros it should invoke and invokes
them. ``single_action`` and ``word_action`` are invoked inside loops.
:mps:tag:`fun.get` :c:func:`BTGet()`. The bit-index will be converted in the usual
way, see :mps:ref:`.index`. The relevant :c:type:`Word` will be read out of the Bit
Table and shifted right by the sub-:c:type:`Word` index (this brings the
relevant bit down to the least significant bit of the :c:type:`Word`), the
:c:type:`Word` will then be masked with 1, producing the answer.
:mps:tag:`fun.set` :c:func:`BTSet()`.
:mps:tag:`fun.res` :c:func:`BTRes()`.
In both :c:func:`BTSet()` and :c:func:`BTRes()` a mask is constructed by shifting 1
left by the sub-word-index (see :mps:ref:`.index`). For :c:func:`BTSet()` the mask is
or-ed into the relevant word (thereby setting a single bit). For
:c:func:`BTRes()` the mask is inverted and and-ed into the relevant word
(thereby resetting a single bit).
:mps:tag:`fun.set-range` :c:func:`BTSetRange()`. :c:func:`ACT_ON_RANGE()` (see :mps:ref:`.iteration`
above) is used with macros that set a single bit (using :c:func:`BTSet()`),
set a range of bits in a word, and set a whole word.
:mps:tag:`fun.res-range` :c:func:`BTResRange()` This is implemented similarly to
:c:func:`BTSetRange()` (:mps:ref:`.fun.set-range`) except using :c:func:`BTRes()` and reverse
bit-masking logic.
:mps:tag:`fun.test.range.set` :c:func:`BTIsSetRange()`. :c:func:`ACT_ON_RANGE()` (see
:mps:ref:`.iteration` above) is used with macros that test whether all the
relevant bits are set; if some of the relevant bits are not set then
``return FALSE`` is used to terminate the iteration early and return
from the :c:func:`BTIsSetRange()` function. If the iteration completes then
:c:macro:`TRUE` is returned.
:mps:tag:`fun.test.range.reset` :c:func:`BTIsResRange()`. As for :c:func:`BTIsSetRange()`
(:mps:ref:`.fun.test.range.set` above) but testing whether the bits are reset.
:mps:tag:`fun.test.range.same` :c:func:`BTRangesSame()`. As for :c:func:`BTIsSetRange()`
(:mps:ref:`.fun.test.range.set` above) but testing whether corresponding
ranges in the two Bit Tables are the same. Note there are no speed
requirements, but :c:func:`ACT_ON_RANGE()` is used for simplicity and
uniformity.
:mps:tag:`fun.find` The four external find functions (:c:func:`BTFindShortResRange()`,
:c:func:`BTFindShortResRangeHigh()`, :c:func:`BTFindLongResRange()`,
:c:func:`BTFindLongResRangeHigh()`) simply call through to one of the two
internal functions: :c:func:`BTFindResRange()` and :c:func:`BTFindResRangeHigh()`.
.. c:function:: Bool BTFindResRange(Index *baseReturn, Index *limitReturn, BT bt, Index searchBase, Index searchLimit, Count minLength, Count maxLength)
.. c:function:: Bool BTFindResRangeHigh(Index *baseReturn, Index *limitReturn, BT bt, Index searchBase, Index searchLimit, Count minLength, Count maxLength)
There are two length parameters, one specifying the minimum length of
the range to be found, the other the maximum length. For
:c:func:`BTFindShort()` and :c:func:`BTFindShortHigh()`, ``maxLength`` is equal to
``minLength`` when passed; for :c:func:`BTFindLong()` and :c:func:`BTFindLongHigh()`,
``maxLength` is equal to the maximum possible range, namely
``searchLimit - searchBase``.
:mps:tag:`fun.find-res-range` :c:func:`BTFindResRange()`. Iterate within the search
boundaries, identifying candidate ranges by searching for a reset bit.
The :ref:`Boyer–Moore algorithm <BM77>` is used (it's particularly
easy to implement when there are only two symbols, 0 and 1, in the
alphabet). For each candidate range, iterate backwards over the bits
from the end of the range towards the beginning. If a set bit is
found, this candidate has failed and a new candidate range is
selected. If when scanning for the set bit a range of reset bits was
found before finding the set bit, then this (small) range of reset
bits is used as the start of the next candidate. Additionally the end
of this small range of reset bits (the end of the failed candidate
range) is remembered so that we don't have to iterate over this range
again. But if no reset bits were found in the candidate range, then
iterate again (starting from the end of the failed candidate) to look
for one. If during the backwards search no set bit is found, then we
have found a sufficiently large range of reset bits; now extend the
valid range as far as possible up to the maximum length by iterating
forwards up to the maximum limit looking for a set bit. The iterations
make use of the :c:func:`ACT_ON_RANGE()` and :c:func:`ACT_ON_RANGE_HIGH()` macros,
which can use ``goto`` to effect an early termination of the iteration
when a set/reset (as appropriate) bit is found. The macro
:c:func:`ACTION_FIND_SET_BIT()` is used in the iterations. It efficiently
finds the first (that is, with lowest index or weight) set bit in a
word or subword.
:mps:tag:`fun.find-res-range.improve` Various other performance improvements
have been suggested in the past, including some from
request.epcore.170534. Here is a list of potential improvements which
all sound plausible, but which have not led to performance
improvements in practice:
* :mps:tag:`fun.find-res-range.improve.step.partial` When the top index in a
candidate range fails, skip partial words as well as whole words,
using, for example, lookup tables.
* :mps:tag:`fun.find-res-range.improve.lookup` When testing a candidate run,
examine multiple bits at once (for example, 8), using lookup tables
for (for example) index of first set bit, index of last set bit,
number of reset bits, length of maximum run of reset bits.
:mps:tag:`fun.find-res-range-high` :c:func:`BTFindResRangeHigh()`. Exactly the same
algorithm as in :c:func:`BTFindResRange()` (see :mps:ref:`.fun.find-res-range` above),
but moving over the table in the opposite direction.
:mps:tag:`fun.copy-simple-range` :c:func:`BTCopyRange()`. Uses :c:func:`ACT_ON_RANGE()` (see
:mps:ref:`.iteration` above) with the obvious implementation. Should be fast.
:mps:tag:`fun.copy-offset-range` :c:func:`BTCopyOffsetRange()`. Uses a simple
iteration loop, reading bits with :c:func:`BTGet()` and setting them with
:c:func:`BTSet()`. Doesn't use :c:func:`ACT_ON_RANGE()` because the two ranges will
not, in general, be similarly word-aligned.
:mps:tag:`fun.copy-invert-range` :c:func:`BTCopyInvertRange()`. Uses :c:func:`ACT_ON_RANGE()`
(see :mps:ref:`.iteration` above) with the obvious implementation. Should be
fast---although there are no speed requirements.
Testing
-------
:mps:tag:`test` The following tests are available or have been used during
development.
:mps:tag:`test.btcv` ``btcv.c``. This is supposed to be a coverage test,
intended to execute all of the module's code in at least some minimal
way.
:mps:tag:`test.cbstest` ``cbstest.c``. This was written as a test of the
:c:macro:`CBS` module (design.mps.cbs(2)). It compares the functional
operation of a :c:macro:`CBS` with that of a :c:type:`BT` so is a good functional
test of either module.
:mps:tag:`test.mmqa.120` MMQA_test_function!210.c. This is used because it has
a fair amount of segment allocation and freeing so exercises the arena
code that uses Bit Tables.
:mps:tag:`test.bttest` ``bttest.c``. This is an interactive test that can be
used to exercise some of the :c:type:`BT` functionality by hand.
:mps:tag:`test.dylan` It is possible to modify Dylan so that it uses Bit
Tables more extensively. See change.mps.epcore.brisling.160181 TEST1
and TEST2.

698
mps/manual/html/_sources/design/buffer.txt Normal file
View file

@ -0,0 +1,698 @@
.. _design-buffer:
.. index::
pair: buffers; design
Allocation buffers and allocation points
========================================
.. mps:prefix:: design.mps.buffer
Introduction
------------
:mps:tag:`scope` This document describes the design of allocation buffers
and allocation points.
:mps:tag:`purpose` The purpose of this document is to record design
decisions made concerning allocation buffers and allocation points and
justify those decisions in terms of requirements.
:mps:tag:`readership` The document is intended for reading by any MPS
developer.
Glossary
--------
trapped
:mps:tag:`def.trapped` The buffer is in a state such that the MPS gets
to know about the next use of that buffer.
Source
------
:mps:tag:`source.mail` Much of the juicy stuff about buffers is only
floating around in mail discussions. You might like to try searching
the archives if you can't find what you want here.
.. note::
Mail archives are only accessible to Ravenbrook staff. RHSK
2006-06-09.
:mps:tag:`source.synchronize` For a discussion of the synchronization
issues:
* `mail.richard.1995-05-19.17-10 </project/mps/mail/1995/05/19/17-10/0.txt>`_
* `mail.ptw.1995-05-19.19-15 </project/mps/mail/1995/05/19/19-15/0.txt>`_
* `mail.richard.1995-05-24.10-18 </project/mps/mail/1995/05/24/10-18/0.txt>`_
.. note::
I believe that the sequence for flip in PTW's message is
incorrect. The operations should be in the other order. DRJ.
:mps:tag:`source.interface` For a description of the buffer interface in C
prototypes:
* `mail.richard.1997-04-28.09-25(0) </project/mps/mail/1997/04/28/09-25/0.txt>`_
:mps:tag:`source.qa` Discussions with QA were useful in pinning down the
semantics and understanding of some obscure but important boundary
cases. See the thread starting
`mail.richard.tucker.1997-05-12.09-45(0)
</project/mps/mail/1997/05/12/09-45/0.txt>`_ with subject "notes on
our allocation points discussion":
* `mail.richard.tucker.1997-05-12.09-45 </project/mps/mail/1997/05/12/09-45/0.txt>`_
* `mail.ptw.1997-05-12.12-46(1) </project/mps/mail/1997/05/12/12-46/1.txt>`_
* `mail.richard.1997-05-12.13-15 </project/mps/mail/1997/05/12/13-15/0.txt>`_
* `mail.richard.1997-05-12.13-28 </project/mps/mail/1997/05/12/13-28/0.txt>`_
* `mail.ptw.1997-05-13.15-15 </project/mps/mail/1997/05/13/15-15/0.txt>`_
* `mail.sheep.1997-05-14.11-52 </project/mps/mail/1997/05/14/11-52/0.txt>`_
* `mail.rit.1997-05-15.09-19 </project/mps/mail/1997/05/15/09-19/0.txt>`_
* `mail.ptw.1997-05-15.21-22 </project/mps/mail/1997/05/15/21-22/0.txt>`_
* `mail.ptw.1997-05-15.21-35 </project/mps/mail/1997/05/15/21-35/0.txt>`_
* `mail.rit.1997-05-16.08-02 </project/mps/mail/1997/05/16/08-02/0.txt>`_
* `mail.rit.1997-05-16.08-42 </project/mps/mail/1997/05/16/08-42/0.txt>`_
* `mail.ptw.1997-05-16.12-36 </project/mps/mail/1997/05/16/12-36/0.txt>`_
* `mail.ptw.1997-05-16.12-47 </project/mps/mail/1997/05/16/12-47/0.txt>`_
* `mail.richard.1997-05-19.15-46 </project/mps/mail/1997/05/19/15-46/0.txt>`_
* `mail.richard.1997-05-19.15-56 </project/mps/mail/1997/05/19/15-56/0.txt>`_
* `mail.ptw.1997-05-20.20-47 </project/mps/mail/1997/05/20/20-47/0.txt>`_
Requirements
------------
:mps:tag:`req.fast` Allocation must be very fast.
:mps:tag:`req.thread-safe` Must run safely in a multi-threaded environment.
:mps:tag:`req.no-synch` Must avoid the use of thread-synchronization.
(:mps:ref:`.req.fast`)
:mps:tag:`req.manual` Support manual memory management.
:mps:tag:`req.exact` Support exact collectors.
:mps:tag:`req.ambig` Support ambiguous collectors.
:mps:tag:`req.count` Must record (approximately) the amount of allocation (in bytes).
.. note::
Actually not a requirement any more, but once was put forward as a
Dylan requirement. Bits of the code still reflect this
requirement. See request.dylan.170554.
Classes
-------
:mps:tag:`class.hierarchy` The :c:type:`Buffer` data structure is designed to be
subclassable (see design.mps.protocol).
:mps:tag:`class.hierarchy.buffer` The basic buffer class (:c:type:`BufferClass`)
supports basic allocation-point buffering, and is appropriate for
those manual pools which don't use segments (:mps:ref:`.req.manual`). The
:c:type:`Buffer` class doesn't support reference ranks (that is, the buffers
have ``RankSetEMPTY``). Clients may use :c:type:`BufferClass` directly, or
create their own subclasses (see :mps:ref:`.subclassing`).
:mps:tag:`class.hierarchy.segbuf` Class :c:type:`SegBufClass` is also provided for
the use of pools which additionally need to associate buffers with
segments. :c:type:`SegBufClass` is a subclass of :c:type:`BufferClass`. Manual
pools may find it convenient to use :c:type:`SegBufClass`, but it is
primarily intended for automatic pools (:mps:ref:`.req.exact`, :mps:ref:`.req.ambig`).
An instance of :c:type:`SegBufClass` may be attached to a region of memory
that lies within a single segment. The segment is associated with the
buffer, and may be accessed with the :c:func:`BufferSeg()` function.
:c:type:`SegBufClass` also supports references at any rank set. Hence this
class or one of its subclasses should be used by all automatic pools
(with the possible exception of leaf pools). The rank sets of buffers
and the segments they are attached to must match. Clients may use
:c:type:`SegBufClass` directly, or create their own subclasses (see
:mps:ref:`.subclassing`).
:mps:tag:`class.hierarchy.rankbuf` Class :c:type:`RankBufClass` is also provided
as a subclass of :c:type:`SegBufClass`. The only way in which this differs
from its superclass is that the rankset of a :c:type:`RankBufClass` is set
during initialization to the singleton rank passed as an additional
parameter to :c:func:`BufferCreate()`. Instances of :c:type:`RankBufClass` are of
the same type as instances of :c:type:`SegBufClass`, that is, ``SegBuf``.
Clients may use :c:type:`RankBufClass` directly, or create their own
subclasses (see :mps:ref:`.subclassing`).
:mps:tag:`class.create` The buffer creation functions (:c:func:`BufferCreate()`
and :c:func:`BufferCreateV()`) take a ``class`` parameter, which determines
the class of buffer to be created.
:mps:tag:`class.choice` Pools which support buffered allocation should
specify a default class for buffers. This class will be used when a
buffer is created in the normal fashion by MPS clients (for example by
a call to :c:func:`mps_ap_create()`). Pools specify the default class by
means of the ``bufferClass`` field in the pool class object. This
should be a pointer to a function of type :c:type:`PoolBufferClassMethod`.
The normal class "Ensure" function (for example
:c:func:`EnsureBufferClass()`) has the appropriate type.
:mps:tag:`subclassing` Pools may create their own subclasses of the standard
buffer classes. This is sometimes useful if the pool needs to add an
extra field to the buffer. The convenience macro
:c:func:`DEFINE_BUFFER_CLASS()` may be used to define subclasses of buffer
classes. See design.mps.protocol.int.define-special.
:mps:tag:`replay` To work with the allocation replayer (see
design.mps.telemetry.replayer), the buffer class has to emit an event
for each call to an external interface, containing all the parameters
passed by the user. If a new event type is required to carry this
information, the replayer (impl.c.eventrep) must then be extended to
recreate the call.
:mps:tag:`replay.pool-buffer` The replayer must also be updated if the
association of buffer class to pool or the buffer class hierarchy is
changed.
:mps:tag:`class.method` Buffer classes provide the following methods (these
should not be confused with the pool class methods related to the
buffer protocol, described in :mps:ref:`.method`):
.. c:type:: Res (*BufferInitMethod)(Buffer buffer, Pool pool, ArgList args)
:mps:tag:`class.method.init` :c:func:`init()` is a class-specific initialization
method called from :c:func:`BufferInit()`. It receives the keyword arguments
passed to to :c:func:`BufferInit()`. Client-defined methods must call their
superclass method (via a next-method call) before performing any
class-specific behaviour. :mps:tag:`replay.init` The :c:func:`init()` method
should emit a ``BufferInit<foo>`` event (if there aren't any extra
parameters, ``<foo> = ""``).
.. c:type:: void (*BufferFinishMethod)(Buffer buffer)
:mps:tag:`class.method.finish` :c:func:`finish()` is a class-specific finish
method called from :c:func:`BufferFinish()`. Client-defined methods must
call their superclass method (via a next-method call) after performing
any class-specific behaviour.
.. c:type:: void (*BufferAttachMethod)(Buffer buffer, Addr base, Addr limit, Addr init, Size size)
:mps:tag:`class.method.attach` :c:func:`attach()` is a class-specific method
called whenever a buffer is attached to memory, via
:c:func:`BufferAttach()`. Client-defined methods must call their superclass
method (via a next-method call) before performing any class-specific
behaviour.
.. c:type:: void (*BufferDetachMethod)(Buffer buffer)
:mps:tag:`class.method.detach` :c:func:`detach()` is a class-specific method
called whenever a buffer is detached from memory, via
:c:func:`BufferDetach()`. Client-defined methods must call their superclass
method (via a next-method call) after performing any class-specific
behaviour.
.. c:type:: Seg (*BufferSegMethod)(Buffer buffer)
:mps:tag:`class.method.seg` :c:func:`seg()` is a class-specific accessor method
which returns the segment attached to a buffer (or :c:macro:`NULL` if there
isn't one). It is called from :c:func:`BufferSeg()`. Clients should not need
to define their own methods for this.
.. c:type:: RankSet (*BufferRankSetMethod)(Buffer buffer)
:mps:tag:`class.method.rankSet` :c:func:`rankSet()` is a class-specific accessor
method which returns the rank set of a buffer. It is called from
:c:func:`BufferRankSet()`. Clients should not need to define their own
methods for this.
.. c:type:: void (*BufferSetRankSetMethod)(Buffer buffer, RankSet rankSet)
:mps:tag:`class.method.setRankSet` :c:func:`setRankSet()` is a class-specific
setter method which sets the rank set of a buffer. It is called from
:c:func:`BufferSetRankSet()`. Clients should not need to define their own
methods for this.
.. c:type:: Res (*BufferDescribeMethod)(Buffer buffer, mps_lib_FILE *stream)
:mps:tag:`class.method.describe` :c:func:`describe()` is a class-specific method
called to describe a buffer, via BufferDescribe. Client-defined
methods must call their superclass method (via a next-method call)
before describing any class-specific state.
Logging
-------
:mps:tag:`logging.control` Buffers have a separate control for whether they
are logged or not, this is because they are particularly high volume.
This is a Boolean flag (``bufferLogging``) in the :c:type:`ArenaStruct`.
Measurement
-----------
:mps:tag:`count` Counting the allocation volume is done by maintaining two
fields in the buffer struct:
:mps:tag:`count.fields` ``fillSize``, ``emptySize``.
:mps:tag:`count.monotonic` both of these fields are monotonically
increasing.
:mps:tag:`count.fillsize` ``fillSize`` is an accumulated total of the size
of all the fills (as a result of calling the :c:type:`PoolClass`
:c:func:`BufferFill()` method) that happen on the buffer.
:mps:tag:`count.emptysize` ``emptySize`` is an accumulated total of the size of
all the empties than happen on the buffer (which are notified to the
pool using the :c:type:`PoolClass` :c:func:`BufferEmpty()` method).
:mps:tag:`count.generic` These fields are maintained by the generic buffer
code in :c:func:`BufferAttach()` and :c:func:`BufferDetach()`.
:mps:tag:`count.other` Similar count fields are maintained in the pool and
the arena. They are maintained on an internal (buffers used internally
by the MPS) and external (buffers used for mutator allocation points)
basis. The fields are also updated by the buffer code. The fields are:
- in the pool: ``fillMutatorSize``, ``fillInternalSize``,
``emptyMutatorSize``, and ``emptyInternalSize`` (4 fields);
- in the arena, ``fillMutatorSize``, ``fillInternalSize``,
``emptyMutatorSize``, ``emptyInternalSize``, and
``allocMutatorSize`` (5 fields).
:mps:tag:`count.alloc.how` The amount of allocation in the buffer just
after an empty is ``fillSize - emptySize``. At other times this
computation will include space that the buffer has the use of (between
base and init) but which may not get allocated in (because the
remaining space may be too large for the next reserve so some or all
of it may get emptied). The arena field ``allocMutatorSize`` is
incremented by the allocated size (between base and init)
whenever a buffer is detached. Symmetrically this field is decremented
by by the pre-allocated size (between base and init) whenever
a buffer is attached. The overall count is asymptotically correct.
:mps:tag:`count.type` All the count fields are type double.
:mps:tag:`count.type.justify` This is because double is the type most likely
to give us enough precision. Because of the lack of genuine
requirements the type isn't so important. It's nice to have it more
precise than long. Which double usually is.
Notes from the whiteboard
-------------------------
Requirements
- atomic update of words
- guarantee order of reads and write to certain memory locations.
Flip
- limit:=0
- record init for scanner
Commit
- init:=alloc
- if(limit = 0) ...
- L written only by MM
- A written only by client (except during synchronized MM op)
- I ditto
- I read by MM during flip
States
- busy
- ready
- trapped
- reset
.. note::
There are many more states. DRJ.
Misc
- During buffer ops all field values can change. Might trash perfectly
good ("valid"?) object if pool isn't careful.
Synchronization
---------------
Buffers provide a loose form of synchronization between the mutator
and the collector.
The crucial synchronization issues are between the operation the pool
performs on flip and the mutator's commit operation.
Commit
- read init
- write init
- Memory Barrier
- read ``limit``
Flip
- write ``limit``
- Memory Barrier
- read init
Commit consists of two parts. The first is the update to init.
This is a declaration that the new object just before init is now
correctly formatted and can be scanned. The second is a check to see
if the buffer has been "tripped". The ordering of the two parts is
crucial.
Note that the declaration that the object is correctly formatted is
independent of whether the buffer has been tripped or not. In
particular a pool can scan up to the init pointer (including the newly
declared object) whether or not the pool will cause the commit to
fail. In the case where the pool scans the object, but then causes the
commit to fail (and presumably the allocation to occur somewhere
else), the pool will have scanned a "dead" object, but this is just
another example of conservatism in the general sense.
Not that the read of init in the Flip sequence can in fact be
arbitrarily delayed (as long as it is read before a buffered segment
is scanned).
On processors with Relaxed Memory Order (such as the DEC Alpha),
Memory Barriers will need to be placed at the points indicated.
::
* DESIGN
*
* design.mps.buffer.
*
* An allocation buffer is an interface to a pool which provides
* very fast allocation, and defers the need for synchronization in
* a multi-threaded environment.
*
* Pools which contain formatted objects must be synchronized so
* that the pool can know when an object is valid. Allocation from
* such pools is done in two stages: reserve and commit. The client
* first reserves memory, then initializes it, then commits.
* Committing the memory declares that it contains a valid formatted
* object. Under certain conditions, some pools may cause the
* commit operation to fail. (See the documentation for the pool.)
* Failure to commit indicates that the whole allocation failed and
* must be restarted. When using a pool which introduces the
* possibility of commit failing, the allocation sequence could look
* something like this:
*
* do {
* res = BufferReserve(&p, buffer, size);
* if(res != ResOK) return res; // allocation fails, reason res
* initialize(p); // p now points at valid object
* } while(!BufferCommit(buffer, p, size));
*
* Pools which do not contain formatted objects can use a one-step
* allocation as usual. Effectively any random rubbish counts as a
* "valid object" to such pools.
*
* An allocation buffer is an area of memory which is pre-allocated
* from a pool, plus a buffer descriptor, which contains, inter
* alia, four pointers: base, init, alloc, and limit. Base points
* to the base address of the area, limit to the last address plus
* one. Init points to the first uninitialized address in the
* buffer, and alloc points to the first unallocated address.
*
* L . - - - - - . ^
* | | Higher addresses -'
* | junk |
* | | the "busy" state, after Reserve
* A |-----------|
* | uninit |
* I |-----------|
* | init |
* | | Lower addresses -.
* B `-----------' v
*
* L . - - - - - . ^
* | | Higher addresses -'
* | junk |
* | | the "ready" state, after Commit
* A=I |-----------|
* | |
* | |
* | init |
* | | Lower addresses -.
* B `-----------' v
*
* Access to these pointers is restricted in order to allow
* synchronization between the pool and the client. The client may
* only write to init and alloc, but in a restricted and atomic way
* detailed below. The pool may read the contents of the buffer
* descriptor at _any_ time. During calls to the fill and trip
* methods, the pool may update any or all of the fields
* in the buffer descriptor. The pool may update the limit at _any_
* time.
*
* Access to buffers by these methods is not synchronized. If a buffer
* is to be used by more than one thread then it is the client's
* responsibility to ensure exclusive access. It is recommended that
* a buffer be used by only a single thread.
*
* [Only one thread may use a buffer at once, unless the client
* places a mutual exclusion around the buffer access in the usual
* way. In such cases it is usually better to create one buffer for
* each thread.]
*
* Here are pseudo-code descriptions of the reserve and commit
* operations. These may be implemented in-line by the client.
* Note that the client is responsible for ensuring that the size
* (and therefore the alloc and init pointers) are aligned according
* to the buffer's alignment.
*
* Reserve(buf, size) ; size must be aligned to pool
* if buf->limit - buf->alloc >= size then
* buf->alloc +=size ; must be atomic update
* p = buf->init
* else
* res = BufferFill(&p, buf, size) ; buf contents may change
*
* Commit(buf, p, size)
* buf->init = buf->alloc ; must be atomic update
* if buf->limit == 0 then
* res = BufferTrip(buf, p, size) ; buf contents may change
* else
* res = True
* (returns True on successful commit)
*
* The pool must allocate the buffer descriptor and initialize it by
* calling BufferInit. The descriptor this creates will fall
* through to the fill method on the first allocation. In general,
* pools should not assign resources to the buffer until the first
* allocation, since the buffer may never be used.
*
* The pool may update the base, init, alloc, and limit fields when
* the fallback methods are called. In addition, the pool may set
* the limit to zero at any time. The effect of this is either:
*
* 1. cause the _next_ allocation in the buffer to fall through to
* the buffer fill method, and allow the buffer to be flushed
* and relocated;
*
* 2. cause the buffer trip method to be called if the client was
* between reserve and commit.
*
* A buffer may not be relocated under other circumstances because
* there is a race between updating the descriptor and the client
* allocation sequence.
Interface
---------
.. c:function:: Res BufferCreate(Buffer *bufferReturn, BufferClass class, Pool pool, Bool isMutator, ArgList args)
:mps:tag:`method.create` Create an allocation buffer in a pool. The buffer
is created in the "ready" state.
A buffer structure is allocated from the space control pool and
partially initialized (in particularly neither the signature nor the
serial field are initialized). The pool class's :c:func:`bufferCreate()`
method is then called. This method can update (some undefined subset
of) the fields of the structure; it should return with the buffer in
the "ready" state (or fail). The remainder of the initialization then
occurs.
If and only if successful then a valid buffer is returned.
.. c:function:: void BufferDestroy(Buffer buffer)
:mps:tag:`method.destroy` Free a buffer descriptor. The buffer must be in
the "ready" state, that is, not between a Reserve and Commit.
Allocation in the area of memory to which the descriptor refers must
cease after :c:func:`BufferDestroy()` is called.
Destroying an allocation buffer does not affect objects which have
been allocated, it just frees resources associated with the buffer
itself.
The pool class's :c:func:`bufferDestroy()` method is called and then the
buffer structure is uninitialized and freed.
.. c:function:: BufferCheck(Buffer buffer)
:mps:tag:`method.check` The check method is straightforward, the non-trivial dependencies checked are:
- The ordering constraints between base, init, alloc, and limit.
- The alignment constraints on base, init, alloc, and limit.
- That the buffer's rank is identical to the segment's rank.
.. c:function:: void BufferAttach(Buffer buffer, Addr base, Addr limit, Addr init, Size size)
:mps:tag:`method.attach` Set the base, init, alloc, and limit fields so that
the buffer is ready to start allocating in area of memory. The alloc
field is set to ``init + size``.
:mps:tag:`method.attach.unbusy` :c:func:`BufferAttach()` must only be applied to
buffers that are not busy.
.. c:function:: void BufferDetach(Buffer buffer, Pool pool)
:mps:tag:`method.detach` Set the seg, base, init, alloc, and limit fields to
zero, so that the next reserve request will call the fill method.
:mps:tag:`method.detach.unbusy` :c:func:`BufferDetach()` must only be applied to
buffers that are not busy.
.. c:function:: Bool BufferIsReset(Buffer buffer)
:mps:tag:`method.isreset` Returns :c:macro:`TRUE` if and only if the buffer is in the
reset state, that is, with base, init, alloc, and limit all set to
zero.
.. c:function:: Bool BufferIsReady(Buffer buffer)
:mps:tag:`method.isready` Returns :c:macro:`TRUE` if and only if the buffer is not
between a reserve and commit. The result is only reliable if the
client is not currently using the buffer, since it may update the
alloc and init pointers asynchronously.
.. c:function:: mps_ap_t (BufferAP)(Buffer buffer)
Returns the :c:type:`APStruct` substructure of a buffer.
.. c:function:: Buffer BufferOfAP(mps_ap_t ap)
:mps:tag:`method.ofap` Return the buffer which owns an :c:type:`APStruct`.
:mps:tag:`method.ofap.thread-safe` :c:func:`BufferOfAP()` must be thread safe (see
impl.c.mpsi.thread-safety). This is achieved simply because the
underlying operation involved is simply a subtraction.
.. c:function:: Arena BufferArena(Buffer buffer)
:mps:tag:`method.arena` Returns the arena which owns a buffer.
:mps:tag:`method.arena.thread-safe` :c:func:`BufferArena()` must be thread safe
(see impl.c.mpsi.thread-safety). This is achieved simple because the
underlying operation is a read of shared-non-mutable data (see
design.mps.thread-safety).
.. c:function:: Pool BufferPool(Buffer buffer)
Returns the pool to which a buffer is attached.
.. c:function:: Res BufferReserve(Addr *pReturn, Buffer buffer, Size size, Bool withReservoirPermit)
:mps:tag:`method.reserve` Reserves memory from an allocation buffer.
This is a provided version of the reserve procedure described above.
The size must be aligned according to the buffer alignment. If
successful, ``ResOK`` is returned and ``*pReturn`` is updated with a
pointer to the reserved memory. Otherwise ``*pReturn`` is not touched.
The reserved memory is not guaranteed to have any particular contents.
The memory must be initialized with a valid object (according to the
pool to which the buffer belongs) and then passed to the
:c:func:`BufferCommit()` method (see below). ``BufferReserve(0`` may not be
applied twice to a buffer without a :c:func:`BufferCommit()` in-between. In
other words, Reserve/Commit pairs do not nest.
.. c:function:: Res BufferFill(Addr *pReturn, Buffer buffer, Size size, Bool withReservoirPermit)
:mps:tag:`method.fill` Refills an empty buffer. If there is not enough space
in a buffer to allocate in-line, :c:func:`BufferFill()` must be called to
"refill" the buffer.
.. c:function:: Bool BufferCommit(Buffer buffer, Addr p, Size size)
:mps:tag:`method.commit` Commit memory previously reserved.
:c:func:`BufferCommit()` notifies the pool that memory which has been
previously reserved (see above) has been initialized with a valid
object (according to the pool to which the buffer belongs). The
pointer ``p`` must be the same as that returned by
:c:func:`BufferReserve()`, and the size must match the size passed to
:c:func:`BufferReserve()`.
:c:func:`BufferCommit()` may not be applied twice to a buffer without a
reserve in between. In other words, objects must be reserved,
initialized, then committed only once.
Commit returns :c:macro:`TRUE` if successful, :c:macro:`FALSE` otherwise. If commit
fails and returns :c:macro:`FALSE`, the client may try to allocate again by
going back to the reserve stage, and may not use the memory at ``p``
again for any purpose.
Some classes of pool may cause commit to fail under rare
circumstances.
.. c:function:: BufferTrip(Buffer buffer, Addr p, Size size)
:mps:tag:`method.trip` Act on a tripped buffer. The pool which owns a buffer
may asynchronously set the buffer limit to zero in order to get
control over the buffer. If this occurs after a :c:func:`BufferReserve()`
(but before the corresponding commit), then the :c:func:`BufferCommit()`
method calls :c:func:`BufferTrip()` and the :c:func:`BufferCommit()` method
returns with the return value of :c:func:`BufferTrip()`.
:mps:tag:`method.trip.precondition` At the time trip is called, from
:c:func:`BufferCommit()`, the following are true:
- :mps:tag:`method.trip.precondition.limit` ``limit == 0``
- :mps:tag:`method.trip.precondition.init` ``init == alloc``
- :mps:tag:`method.trip.precondition.p` ``p + size == alloc``
Diagrams
--------
Here are a number of diagrams showing how buffers behave. In general,
the horizontal axis corresponds to mutator action (reserve, commit)
and the vertical axis corresponds to collector action. I'm not sure
which of the diagrams are the same as each other, and which are best
or most complete when they are different, but they all attempt to show
essentially the same information. It's very difficult to get all the
details in. These diagrams were drawn by Richard Brooksby, Richard
Tucker, Gavin Matthews, and others in April 1997. In general, the
later diagrams are, I suspect, more correct, complete and useful than
the earlier ones. I have put them all here for the record. Richard
Tucker, 1998-02-09.
Buffer Diagram:
Buffer States
Buffer States (3-column)
Buffer States (4-column)
Buffer States (gavinised)
Buffer States (interleaved)
Buffer States (richardized)
[missing diagrams]

695
mps/manual/html/_sources/design/cbs.txt
View file

@ -1,695 +0,0 @@
.. sources:
`<https://info.ravenbrook.com/project/mps/master/design/cbs/>`_
.. mps:prefix:: design.mps.cbs
Coalescing block structure
==========================
Introduction
------------
:mps:tag:`intro` This is the design for :mps:ref:`impl.c.cbs`, which
implements a data structure for the management of non-intersecting
memory ranges, with eager coalescence.
:mps:tag:`readership` This document is intended for any MM developer.
:mps:tag:`source` :mps:ref:`design.mps.poolmv2`, :mps:ref:`design.mps.poolmvff`.
:mps:tag:`overview` The "coalescing block structure" is a set of
addresses (or a subset of address space), with provision for efficient
management of contiguous ranges, including insertion and deletion,
high level communication with the client about the size of contiguous
ranges, and detection of protocol violations.
History
-------
:mps:tag:`hist.0` This document was derived from the outline in
:mps:ref:`design.mps.poolmv2(2)`. Written by Gavin Matthews
1998-05-01.
:mps:tag:`hist.1` Updated by Gavin Matthews 1998-07-22 in response to
approval comments in :mps:ref:`change.epcore.anchovy.160040` There is
too much fragmentation in trapping memory.
:mps:tag:`hist.2` Updated by Gavin Matthews (as part of
:mps:ref:`change.epcore.brisling.160158`: MVFF cannot be instantiated
with 4-byte alignment) to document new alignment restrictions.
:mps:tag:`hist.3` Converted from MMInfo database design document.
Richard Brooksby, 2002-06-07.
:mps:tag:`hist.4` Converted to reStructuredText. Gareth Rees,
2013-04-14.
Definitions
-----------
:mps:tag:`def.range` A (contiguous) range of addresses is a semi-open
interval on address space.
:mps:tag:`def.isolated` A contiguous range is isolated with respect to
some property it has, if adjacent elements do not have that property.
:mps:tag:`def.interesting` A block is interesting if it is of at least
the minimum interesting size specified by the client.
Requirements
------------
:mps:tag:`req.set` Must maintain a set of addresses.
:mps:tag:`req.fast` Common operations must have a low amortized cost.
:mps:tag:`req.add` Must be able to add address ranges to the set.
:mps:tag:`req.remove` Must be able to remove address ranges from the set.
:mps:tag:`req.size` Must report concisely to the client when isolated
contiguous ranges of at least a certain size appear and disappear.
:mps:tag:`req.iterate` Must support the iteration of all isolated
contiguous ranges. This will not be a common operation.
:mps:tag:`req.protocol` Must detect protocol violations.
:mps:tag:`req.debug` Must support debugging of client code.
:mps:tag:`req.small` Must have a small space overhead for the storage
of typical subsets of address space and not have abysmal overhead for
the storage of any subset of address space.
:mps:tag:`req.align` Must support an alignment (the alignment of all
addresses specifying ranges) of down to ``sizeof(void *)`` without
losing memory.
Interface
---------
:mps:tag:`header` CBS is used through :mps:ref:`impl.h.cbs`.
External types
..............
.. c:type:: typedef struct CBSStruct CBSStruct, *CBS;
:mps:tag:`type.cbs` :c:type:`CBS` is the main datastructure for
manipulating a CBS. It is intended that a :c:type:`CBSStruct` be
embedded in another structure. No convenience functions are provided
for the allocation or deallocation of the CBS.
.. c:type:: typedef struct CBSBlockStruct CBSBlockStruct, *CBSBlock;
:mps:tag:`type.cbs.block` :c:type:`CBSBlock` is the data-structure
that represents an isolated contiguous range held by the CBS. It is
returned by the new and delete methods described below.
:mps:tag:`type.cbs.method` The following methods are provided as
callbacks to advise the client of certain events. The implementation
of these functions should not cause any CBS function to be called on
the same CBS. In this respect, the CBS module is not re-entrant.
.. c:type:: typedef void (*CBSChangeSizeMethod)(CBS cbs, CBSBlock block, Size oldSize, SizeNewSize);
:mps:tag:`type.cbs.change.size.method` :c:type:`CBSChangeSizeMethod`
is the function pointer type, four instances of which are optionally
registered via CBSInit.
These callbacks are invoked under :c:func:`CBSInsert`,
:c:func:`CBSDelete`, or :c:func:`CBSSetMinSize` in certain
circumstances. Unless otherwise stated, ``oldSize`` and ``newSize``
will both be non-zero, and different. The accessors
:c:func:`CBSBlockBase`, :c:func:`CBSBlockLimit`, and
:c:func:`CBSBlockSize` may be called from within these callbacks,
except within the delete callback when ``newSize`` is zero. See
:mps:ref:`.impl.callback` for implementation details.
.. c:type:: typedef Bool (*CBSIterateMethod)(CBS cbs, CBSBlock block, void *closureP, unsigned long closureS);
:mps:tag:`type.cbs.iterate.method` :c:type:`CBSIterateMethod` is a
function pointer type for a client method invoked by the CBS module
for every isolated contiguous range in address order, when passed to
the :c:func:`CBSIterate` or :c:func:`CBSIterateLarge` functions. The
function returns a boolean indicating whether to continue with the
iteration.
External functions
..................
.. c:function:: Res CBSInit(Arena arena, CBS cbs, CBSChangeSizeMethod new, CBSChangeSizeMethod delete, CBSChangeSizeMethod grow, CBSChangeSizeMethod shrink, Size minSize, Align alignment, Bool mayUseInline)
:mps:tag:`function.cbs.init` :c:func:`CBSInit` is the function that
initialises the CBS structure. It performs allocation in the supplied
arena. Four methods are passed in as function pointers (see
:mps:ref:`.type.*` above), any of which may be ``NULL``. It receives a
minimum size, which is used when determining whether to call the
optional methods. The ``mayUseInline`` Boolean indicates whether the
CBS may use the memory in the ranges as a low-memory fallback (see
:mps:ref:`.impl.low-mem`). The alignment indicates the alignment of
ranges to be maintained. An initialised CBS contains no ranges.
:mps:tag:`function.cbs.init.may-use-inline` If ``mayUseInline`` is
set, then ``alignment`` must be at least ``sizeof(void *)``. In this
mode, the CBS will never fail to insert or delete ranges, even if
memory for control structures becomes short. Note that, in such cases,
the CBS may defer notification of new/grow events, but will report
available blocks in :c:func:`CBSFindFirst` and :c:func:`CBSFindLast`.
Such low memory conditions will be rare and transitory. See
:mps:ref:`.align` for more details.
.. c:function:: void CBSFinish(CBS cbs)
:mps:tag:`function.cbs.finish` :c:func:`CBSFinish` is the function
that finishes the CBS structure and discards any other resources
associated with the CBS.
.. c:function:: Res CBSInsert(CBS cbs, Addr base, Addr limit)
:mps:tag:`function.cbs.insert` :c:func:`CBSInsert` is the function
used to add a contiguous range specified by ``[base,limit)`` to the
CBS. If any part of the range is already in the CBS, then
:c:macro:`ResFAIL` is returned, and the CBS is unchanged. This
function may cause allocation; if this allocation fails, and any
contingency mechanism fails, then :c:macro:`ResMEMORY` is returned,
and the CBS is unchanged.
:mps:tag:`function.cbs.insert.callback` :c:func:`CBSInsert` will invoke callbacks as follows:
* ``new``: when a new block is created that is interesting. ``oldSize == 0; newSize >= minSize``.
* ``new``: when an uninteresting block coalesces to become interesting. ``0 < oldSize < minSize <= newSize``.
* ``delete``: when two interesting blocks are coalesced. ``grow`` will also be invoked in this case on the larger of the two blocks. ``newSize == 0; oldSize >= minSize``.
* ``grow``: when an interesting block grows in size. ``minSize <= oldSize < newSize``.
.. c:function:: Res CBSDelete(CBS cbs, Addr base, Addr limit)
:mps:tag:`function.cbs.delete` :c:func:`CBSDelete` is the function
used to remove a contiguous range specified by ``[base,limit)`` from
the CBS. If any part of the range is not in the CBS, then
:c:macro:`ResFAIL` is returned, and the CBS is unchanged. This
function may cause allocation; if this allocation fails, and any
contingency mechanism fails, then :c:macro:`ResMEMORY` is returned,
and the CBS is unchanged.
:mps:tag:`function.cbs.delete.callback` :c:func:`CBSDelete` will
invoke callbacks as follows:
* ``delete``: when an interesting block is entirely removed. ``newSize == 0; oldSize >= minSize``.
* ``delete``: when an interesting block becomes uninteresting. ``0 < newSize < minSize <= oldSize``.
* ``new``: when a block is split into two blocks, both of which are interesting. ``shrink`` will also be invoked in this case on the larger of the two blocks. ``oldSize == 0; newSize >= minSize``.
* ``shrink``: when an interesting block shrinks in size, but remains interesting. ``minSize <= newSize < oldSize``.
.. c:function:: void CBSIterate(CBS cbs, CBSIterateMethod iterate, void *closureP, unsigned long closureS)
:mps:tag:`function.cbs.iterate` :c:func:`CBSIterate` is the function
used to iterate all isolated contiguous ranges in a CBS. It receives a
pointer, unsigned long closure pair to pass on to the iterator method,
and an iterator method to invoke on every range in address order. If
the iterator method returns ``FALSE``, then the iteration is
terminated.
.. c:function:: void CBSIterateLarge(CBS cbs, CBSIterateMethod iterate, void *closureP, unsigned long closureS)
:mps:tag:`function.cbs.iterate.large` :c:func:`CBSIterateLarge` is the
function used to iterate all isolated contiguous ranges of size
greater than or equal to the client indicated minimum size in a CBS.
It receives a pointer, unsigned long closure pair to pass on to the
iterator method, and an iterator method to invoke on every large range
in address order. If the iterator method returns ``FALSE``, then the
iteration is terminated.
.. c:function:: void CBSSetMinSize(CBS cbs, Size minSize)
:mps:tag:`function.cbs.set.min-size` :c:func:`CBSSetMinSize` is the
function used to change the minimum size of interest in a CBS. This
minimum size is used to determine whether to invoke the client
callbacks from :c:func:`CBSInsert` and :c:func:`CBSDelete`. This
function will invoke either the ``new`` or ``delete`` callback for all
blocks that are (in the semi-open interval) between the old and new
values. ``oldSize`` and ``newSize`` will be the same in these cases.
.. c:function:: Res CBSDescribe(CBS cbs, mps_lib_FILE *stream)
:mps:tag:`function.cbs.describe` :c:func:`CBSDescribe` is a function
that prints a textual representation of the CBS to the given stream,
indicating the contiguous ranges in order, as well as the structure of
the underlying splay tree implementation. It is provided for debugging
purposes only.
.. c:function:: Addr CBSBlockBase(CBSBlock block)
:mps:tag:`function.cbs.block.base` The :c:func:`CBSBlockBase` function
returns the base of the range represented by the :c:type:`CBSBlock`.
This function may not be called from the delete callback when the
block is being deleted entirely.
.. note::
The value of the base of a particular :c:type:`CBSBlock` is not
guaranteed to remain constant across calls to :c:func:`CBSDelete`
and :c:func:`CBSInsert`, regardless of whether a callback is
invoked.
.. c:function:: Addr CBSBlockLimit(CBSBlock block)
:mps:tag:`function.cbs.block.limit` The :c:func:`CBSBlockLimit`
function returns the limit of the range represented by the
:c:type:`CBSBlock`. This function may not be called from the delete
callback when the block is being deleted entirely.
.. note::
The value of the limit of a particular :c:type:`CBSBlock` is not
guaranteed to remain constant across calls to :c:func:`CBSDelete`
and :c:func:`CBSInsert`, regardless of whether a callback is
invoked.
.. c:function:: Size CBSBlockSize(CBSBlock block)
:mps:tag:`function.cbs.block.size` The :c:func:`CBSBlockSize` function
returns the size of the range represented by the :c:type:`CBSBlock`.
This function may not be called from the ``delete`` callback when the
block is being deleted entirely.
.. note::
The value of the size of a particular :c:type:`CBSBlock` is not
guaranteed to remain constant across calls to :c:func:`CBSDelete`
and :c:func:`CBSInsert`, regardless of whether a callback is
invoked.
.. c:function:: Res CBSBlockDescribe(CBSBlock block, mps_lib_FILE *stream)
:mps:tag:`function.cbs.block.describe` The :c:func:`CBSBlockDescribe`
function prints a textual representation of the :c:type:`CBSBlock` to
the given stream. It is provided for debugging purposes only.
.. c:function:: Bool CBSFindFirst(Addr *baseReturn, Addr *limitReturn, CBS cbs, Size size, CBSFindDelete findDelete)
:mps:tag:`function.cbs.find.first` The :c:func:`CBSFindFirst` function
locates the first block (in address order) within the CBS of at least
the specified size, and returns its range. If there are no such
blocks, it returns ``FALSE``. It optionally deletes the top, bottom,
or all of the found range, depending on the ``findDelete`` argument
(this saves a separate call to :c:func:`CBSDelete`, and uses the
knowledge of exactly where we found the range), which must come from
this enumeration::
enum {
CBSFindDeleteNONE, /* don't delete after finding */
CBSFindDeleteLOW, /* delete precise size from low end */
CBSFindDeleteHIGH, /* delete precise size from high end */
CBSFindDeleteENTIRE /* delete entire range */
};
.. c:function:: Bool CBSFindLast(Addr *baseReturn, Addr *limitReturn, CBS cbs, Size size, CBSFindDelete findDelete)
:mps:tag:`function.cbs.find.last` The :c:func:`CBSFindLast` function
locates the last block (in address order) within the CBS of at least
the specified size, and returns its range. If there are no such
blocks, it returns ``FALSE``. Like :c:func:`CBSFindFirst`, it
optionally deletes the range.
.. c:function:: Bool CBSFindLargest(Addr *baseReturn, Addr *limitReturn, CBS cbs, CBSFindDelete findDelete)
:mps:tag:`function.cbs.find.largest` The :c:func:`CBSFindLargest`
function locates the largest block within the CBS, and returns its
range. If there are no blocks, it returns ``FALSE``. Like
:c:func:`CBSFindFirst`, it optionally deletes the range (specifying
``CBSFindDeleteLOW`` or ``CBSFindDeleteHIGH`` has the same effect as
``CBSFindDeleteENTIRE``).
Alignment
---------
:mps:tag:`align` When ``mayUseInline`` is specified to permit inline
data structures and hence avoid losing memory in low memory
situations, the alignments that the CBS supports are constrained by
three requirements:
- The smallest possible range (namely one that is the alignment in
size) must be large enough to contain a single ``void *`` pointer (see
:mps:ref:`.impl.low-mem.inline.grain`);
- Any larger range (namely one that is at least twice the alignment in
size) must be large enough to contain two ``void *`` pointers (see
:mps:ref:`.impl.low-mem.inline.block`);
- It must be valid on all platforms to access a ``void *`` pointer
stored at the start of an aligned range.
All alignments that meet these requirements are aligned to
``sizeof(void *)``, so we take that as the minimum alignment.
Implementation
--------------
:mps:tag:`impl` Note that this section is concerned with describing
various aspects of the implementation. It does not form part of the
interface definition.
Size change callback protocol
.............................
:mps:tag:`impl.callback` The size change callback protocol concerns
the mechanism for informing the client of the appearance and
disappearance of interesting ranges. The intention is that each range
has an identity (represented by the :c:type:`CBSBlock`). When blocks
are split, the larger fragment retains the identity. When blocks are
merged, the new block has the identity of the larger fragment.
:mps:tag:`impl.callback.delete` Consider the case when the minimum
size is ``minSize``, and :c:func:`CBSDelete` is called to remove a
range of size ``middle``. The two (possibly non-existant) neighbouring
ranges have (possibly zero) sizes ``left`` and ``right``. ``middle`` is part
of the :c:type:`CBSBlock` ``middleBlock``.
:mps:tag:`impl.callback.delete.delete` The ``delete`` callback will be
called in this case if and only if::
left + middle + right >= minSize && left < minSize && right < minSize
That is, the combined range is interesting, but neither remaining
fragment is. It will be called with the following parameters:
* ``block``: ``middleBlock``
* ``oldSize``: ``left + middle + right``
* ``newSize``: ``left >= right ? left : right``
:mps:tag:`impl.callback.delete.new` The ``new`` callback will be
called in this case if and only if::
left >= minSize && right >= minSize
That is, both remaining fragments are interesting. It will be called
with the following parameters:
* ``block``: a new block
* ``oldSize``: ``0``
* ``newSize``: ``left >= right ? right : left``
:mps:tag:`impl.callback.delete.shrink` The shrink callback will be
called in this case if and only if::
left + middle + right >= minSize && (left >= minSize || right >= minSize)
That is, at least one of the remaining fragments is still interesting. It will be called with the following parameters:
* ``block``: ``middleBlock``
* ``oldSize``: ``left + middle + right``
* ``newSize``: ``left >= right ? left : right``
:mps:tag:`impl.callback.insert` Consider the case when the minimum
size is ``minSize``, and :c:func:`CBSInsert` is called to add a range
of size ``middle``. The two (possibly non-existant) neighbouring
blocks are ``leftBlock`` and ``rightBlock``, and have (possibly zero)
sizes ``left`` and ``right``.
:mps:tag:`impl.callback.insert.delete` The ``delete`` callback will be
called in this case if and only if:
left >= minSize && right >= minSize
That is, both neighbours were interesting. It will be called with the
following parameters:
* ``block``: ``left >= right ? rightBlock : leftBlock``
* ``oldSize``: ``left >= right ? right : left``
* ``newSize``: ``0``
:mps:tag:`impl.callback.insert.new` The ``new`` callback will be
called in this case if and only if:
left + middle + right >= minSize && left < minSize && right < minSize
That is, the combined block is interesting, but neither neighbour was.
It will be called with the following parameters:
* ``block``: ``left >= right ? leftBlock : rightBlock``
* ``oldSize``: ``left >= right ? left : right``
* ``newSize``: ``left + middle + right``
:mps:tag:`impl.callback.insert.grow` The ``grow`` callback will be
called in this case if and only if::
left + middle + right >= minSize && (left >= minSize || right >= minSize)
That is, at least one of the neighbours was interesting. It will be
called with the following parameters:
* ``block``: ``left >= right ? leftBlock : rightBlock``
* ``oldSize``: ``left >= right ? left : right``
* ``newSize``: ``left + middle + right``
Splay tree
..........
:mps:tag:`impl.splay` The CBS is principally implemented using a splay
tree (see :mps:ref:`design.mps.splay`). Each splay tree node is
embedded in a CBSBlock that represents a semi-open address range. The
key passed for comparison is the base of another range.
:mps:tag:`impl.splay.fast-find` :c:func:`CBSFindFirst` and
:c:func:`CBSFindLast` use the update/refresh facility of splay trees
to store, in each :c:type:`CBSBlock`, an accurate summary of the
maximum block size in the tree rooted at the corresponding splay node.
This allows rapid location of the first or last suitable block, and
very rapid failure if there is no suitable block.
:mps:tag:`impl.find-largest` :c:func:`CBSFindLargest` simply finds out
the size of the largest block in the CBS from the root of the tree
(using :c:func:`SplayRoot`), and does :c:func:`SplayFindFirst` for a
block of that size. This is O(log(*n*)) in the size of the free list,
so it's about the best you can do without maintaining a separate
priority queue, just to do :c:func:`CBSFindLargest`. Except when the
emergency lists (see :mps:ref:`.impl.low-mem`) are in use, they are
also searched.
Low memory behaviour
....................
:mps:tag:`impl.low-mem` Low memory situations cause problems when the
CBS tries to allocate a new :c:type:`CBSBlock` structure for a new
isolated range as a result of either :c:func:`CBSInsert` or
:c:func:`CBSDelete`, and there is insufficient memory to allocation
the :c:type:`CBSBlock` structure:
:mps:tag:`impl.low-mem.no-inline` If ``mayUseInline`` is ``FALSE``,
then the range is not added to the CBS, and the call to
:c:func:`CBSInsert` or :c:func:`CBSDelete` returns ``ResMEMORY``.
:mps:tag:`impl.low-mem.inline` If ``mayUseInline`` is ``TRUE``:
:mps:tag:`impl.low-mem.inline.block` If the range is large enough to
contain an inline block descriptor consisting of two pointers, then it
is kept on an emergency block list. The CBS will eagerly attempt to
add this block back into the splay tree during subsequent calls to
:c:func:`CBSInsert` and :c:func:`CBSDelete`. The CBS will also keep
its emergency block list in address order, and will coalesce this list
eagerly. Some performance degradation will be seen when the emergency
block list is in use. Ranges on this emergency block list will not be
made available to the CBS's client via callbacks. :c:func:`CBSIterate`
and :c:func:`CBSIterateLarge` will not iterate over ranges on this
list.
:mps:tag:`impl.low-mem.inline.block.structure` The two pointers stored
are to the next such block (or ``NULL``), and to the limit of the
block, in that order.
:mps:tag:`impl.low-mem.inline.grain` Otherwise, the range must be
large enough to contain an inline grain descriptor consisting of one
pointer, then it is kept on an emergency grain list. The CBS will
eagerly attempt to add this grain back into either the splay tree or
the emergency block list during subsequent calls to
:c:func:`CBSInsert` and :c:func:`CBSDelete`. The CBS will also keep
its emergency grain list in address order. Some performance
degradation will be seen when the emergency grain list is in use.
Ranges on this emergency grain list will not be made available to the
CBS's client via callbacks. :c:func:`CBSIterate` and
:c:func:`CBSIterateLarge` will not iterate over ranges on this list.
:mps:tag:`impl.low-mem.inline.grain.structure` The pointer stored is
to the next such grain, or ``NULL``.
The CBS block
.............
:mps:tag:`impl.cbs.block` The block contains a base-limit pair and a
splay tree node.
:mps:tag:`impl.cbs.block.special` The base and limit may be equal if
the block is halfway through being deleted.
:mps:tag:`impl.cbs.block.special.just` This conflates values and
status, but is justified because block size is very important.
Testing
-------
:mps:tag:`test` The following testing will be performed on this module:
:mps:tag:`test.cbstest` There is a stress test for this module in
:mps:ref:`impl.c.cbstest`. This allocates a large block of memory and
then simulates the allocation and deallocation of ranges within this
block using both a :c:type:`CBS` and a :c:type:`BT`. It makes both
valid and invalid requests, and compares the :c:type:`CBS` response to
the correct behaviour as determined by the :c:type:`BT`. It also
iterates the ranges in the :c:type:`CBS`, comparing them to the
:c:type:`BT`. It also invokes the :c:func:`CBSDescribe` method, but
makes no automatic test of the resulting output. It does not currently
test the callbacks.
:mps:tag:`test.pool` Several pools (currently :ref:`pool-mvt` and
:ref:`pool-mvff`) are implemented on top of a CBS. These pool are
subject to testing in development, QA, and are/will be heavily
exercised by customers.
Notes for future development
----------------------------
:mps:tag:`future.not-splay` The initial implementation of CBSs is
based on splay trees. It could be revised to use any other data
structure that meets the requirements (especially
:mps:ref:`.req.fast`).
:mps:tag:`future.hybrid` It would be possible to attenuate the problem
of :mps:ref:`.risk.overhead` (below) by using a single word bit set to
represent the membership in a (possibly aligned) word-width of grains.
This might be used for block sizes less than a word-width of grains,
converting them when they reach all free in the bit set. Note that
this would make coalescence slightly less eager, by up to
``(word-width - 1)``.
Risks
-----
:mps:tag:`risk.overhead` Clients should note that the current
implementation of CBSs has a space overhead proportional to the number
of isolated contiguous ranges. [Four words per range.] If the CBS
contains every other grain in an area, then the overhead will be large
compared to the size of that area. [Four words per two grains.] See
:mps:ref:`.future.hybrid` for a suggestion to solve this problem. An
alternative solution is to use CBSs only for managing long ranges.
Proposed hybrid implementation
------------------------------
.. note::
The following relates to a pending re-design and does not yet
relate to any working source version. GavinM 1998-09-25
The CBS system provides its services by combining the services
provided by three subsidiary CBS modules:
- ``CBSST`` -- Splay Tree: Based on out-of-line splay trees; must
allocate to insert isolated, which may therefore fail.
- ``CBSBL`` -- Block List: Based on a singly-linked list of variable
sized ranges with inline descriptors; ranges must be at least large
enough to store the inline descriptor.
- ``CBSGL`` -- Grain List: Based on a singly-linked list of fixed size
ranges with inline descriptors; the ranges must be the alignment of
the CBS.
The three sub-modules have a lot in common. Although their methods are
not invoked via a dispatcher, they have been given consistent
interfaces, and consistent internal appearance, to aid maintenance.
Methods supported by sub-modules (not all sub-modules support all
methods):
- ``MergeRange`` -- Finds any ranges in the specific CBS adjacent to
the supplied one. If there are any, it extends the ranges, possibly
deleting one of them. This cannot fail, but should return ``FALSE``
if there is an intersection between the supplied range and a range
in the specific CBS.
- ``InsertIsolatedRange`` -- Adds a range to the specific CBS that is
not adjacent to any range already in there. Depending on the
specific CBS, this may be able to fail for allocation reasons, in
which case it should return ``FALSE``. It should :c:func:`AVER` if
the range is adjacent to or intersects with a range already there.
- ``RemoveAdjacentRanges`` -- Finds and removes from the specific CBS
any ranges that are adjacent to the supplied range. Should return
``FALSE`` if the supplied range intersects with any ranges already
there.
- ``DeleteRange`` -- Finds and deletes the supplied range from the
specific CBS. Returns a tri-state result:
- ``Success`` -- The range was successfully deleted. This may have
involved the creation of a new range, which should be done via
``CBSInsertIsolatedRange``.
- ``ProtocolError`` -- Either some non-trivial strict subset of the
supplied range was in the specific CBS, or a range adjacent to the
supplied range was in the specific CBS. Either of these indicates
a protocol error.
- ``NoIntersection`` -- The supplied range was not found in the CBS.
This may or not be a protocol error, depending on the invocation
context.
- ``FindFirst`` -- Returns the first (in address order) range in the
specific CBS that is at least as large as the supplied size, or
``FALSE`` if there is no such range.
- ``FindFirstBefore`` -- As ``FindFirst``, but only finds ranges prior
to the supplied address.
- ``FindLast`` -- As ``FindFirst``, but finds the last such range in
address order.
- ``FindLastAfter`` -- ``FindLast`` equivalent of ``FindFirstBefore``.
- ``Init`` -- Initialise the control structure embedded in the CBS.
- ``Finish`` -- Finish the control structure embedded in the CBS.
- ``InlineDescriptorSize`` -- Returns the aligned size of the inline descriptor.
- ``Check`` -- Checks the control structure embedded in the CBS.
The CBS supplies the following utilities:
- ``CBSAlignment`` -- Returns the alignment of the CBS.
- ``CBSMayUseInline`` -- Returns whether the CBS may use the memory in
the ranges stored.
- ``CBSInsertIsolatedRange`` -- Wrapper for ``CBS*InsertIsolatedRange``.
Internally, the ``CBS*`` sub-modules each have an internal structure
``CBS*Block`` that represents an isolated range within the module. It
supports the following methods (for sub-module internal use):
- ``BlockBase`` -- Returns the base of the associated range;
- ``BlockLimit``
- ``BlockRange``
- ``BlockSize``

107
mps/manual/html/_sources/design/check.txt
View file

@ -1,6 +1,109 @@
.. _design-check:
.. index::
pair: checking; design
.. _design-check:
.. include:: ../../converted/check.rst
Checking
========
.. mps:prefix:: design.mps.check
Introduction
------------
:mps:tag:`intro` This documents the design of structure checking within the
MPS.
:mps:tag:`readership` MPS developers.
Implementation
--------------
:mps:tag:`level` There are three levels of checking:
1. :mps:tag:`level.sig` The lowest level checks only that the
structure has a valid ``Signature`` (see
design.mps.sig).
2. :mps:tag:`level.shallow` Shallow checking checks all local fields
(including signature) and also checks the signatures of any parent
or child structures.
3. :mps:tag:`level.deep` Deep checking checks all local fields
(including signatures), the signatures of any parent structures,
and does full recursive checking on any child structures.
:mps:tag:`level.control` Control over the levels of checking is via the
definition of at most one of the macros :c:macro:`TARGET_CHECK_SHALLOW`
(which if defined gives :mps:ref:`.level.shallow`), :c:macro:`TARGET_CHECK_DEEP`
(which if defined gives :mps:ref:`.level.deep`). If neither macro is defined
then :mps:ref:`.level.sig` is used. These macros are not intended to be
manipulated directly by developers, they should use the interface in
impl.h.target.
:mps:tag:`order` Because deep checking (:mps:ref:`.level.deep`) uses unchecked
recursion, it is important that child relationships are acyclic
(:mps:ref:`.macro.down`).
:mps:tag:`fun` Every abstract data type which is a structure pointer should
have a function ``<type>Check`` which takes a pointer of type
``<type>`` and returns a :c:type:`Bool`. It should check all fields in
order, using one of the macros in :mps:ref:`.macro`, or document why not.
:mps:tag:`fun.omit` The only fields which should be omitted from a check
function are those for which there is no meaningful check (for
example, an unlimited unsigned integer with no relation to other
fields).
:mps:tag:`fun.return` Although the function returns a :c:type:`Bool`, if the assert
handler returns (or there is no assert handler), then this is taken to
mean "ignore and continue", and the check function hence returns
:c:macro:`TRUE`.
:mps:tag:`macro` Checking is implemented by invoking four macros in
impl.h.assert:
.. c:function:: CHECKS(type, val)
:mps:tag:`macro.sig` ``CHECKS(type, val)`` checks the signature only, and
should be called precisely on ``type`` and the received object
pointer.
.. c:function:: CHECKL(cond)
:mps:tag:`macro.local` ``CHECKL(cond)`` checks a local field (depending on
level; see :mps:ref:`.level`), and should be called on each local field that
is not an abstract data type structure pointer itself (apart from
the signature), with an appropriate normally-true test condition.
.. c:function:: CHECKU(type, val)
:mps:tag:`macro.up` ``CHECKU(type, val)`` checks a parent abstract data
type structure pointer, performing at most signature checks
(depending on level; see :mps:ref:`.level`). It should be called with the
parent type and pointer.
.. c:function:: CHECKD(type, val)
:mps:tag:`macro.down` ``CHECKD(type, val)`` checks a child abstract data
type structure pointer, possibly invoking ``<type>Check`` (depending
on level; see :mps:ref:`.level`). It should be called with the child type
and pointer.
:mps:tag:`full-type` :c:func:`CHECKS()`, :c:func:`CHECKD()`, :c:func:`CHECKU()`, all operate
only on fully fledged types. This means the type has to provide a
function ``Bool TypeCheck(Type type)`` where ``Type`` is substituted
for the name of the type (for example, :c:func:`PoolCheck()`), and the
expression ``obj->sig`` must be a valid value of type :c:type:`Sig` whenever
``obj`` is a valid value of type ``Type``.
:mps:tag:`type.no-sig` This tag is to be referenced in implementations
whenever the form ``CHECKL(ThingCheck(thing))`` is used instead of
``CHECK{U,D}(Thing, thing)`` because ``Thing`` is not a fully fledged
type (:mps:ref:`.full-type`).

259
mps/manual/html/_sources/design/class-interface.txt
View file

@ -1,6 +1,261 @@
.. _design-class-interface:
.. index::
pair: class interface; design
.. _design-class-interface:
.. include:: ../../converted/class-interface.rst
Pool class interface
====================
.. mps:prefix:: design.mps.class-interface
Introduction
-------------
:mps:tag:`intro` This document describes the interface and protocols between
the MPM and the pool class implementations.
.. note::
This document should be merged into design.mps.pool. Pekka P.
Pirinen, 1999-07-20.
Methods
-------
:mps:tag:`methods` These methods are provided by pool classes as part of the
:c:type:`PoolClass` object (see impl.h.mpmst.class). They form the interface
which allows the MPM to treat pools in a uniform manner.
The following description is based on the definition of the
:c:type:`PoolClassStruct` (impl.h.mpmst.class).
If a class is not required to provide a certain method then it should
set the appropriate ``PoolNo*`` method for that method. It is not
acceptable to use :c:macro:`NULL`.
.. note::
There are also some ``PoolTriv*`` methods. David Jones, 1997-08-19.
:mps:tag:`method.name` The name field should be a short, pithy, cryptic name
for the pool class. Examples are "AMC", "MV".
The ``size`` field is the size of the pool instance structure. For the
``Foo`` :c:type:`PoolClass` this can reasonably be expected to be
``sizeof(FooStruct)``.
The ``offset`` field is the offset into the pool instance structure of
the generic :c:type:`PoolStruct`. Typically this field is called
``poolStruct``, so something like ``offsetof(FooStruct, poolStruct)``
is typical. If possible, arrange for this to be zero.
The ``init`` field is the class's init method. This method is called
via the generic function :c:func:`PoolInit()`, which is in turn called by
:c:func:`PoolCreate()`. The generic function allocates the pool's structure
(using the size and offset information), initializes the
:c:type:`PoolStruct` (generic part) then calls the ``init`` method to do any
class-specific initialization. Typically this means initializing the
fields in the class instance structure. If ``init`` returns a non-OK
result code the instance structure will be deallocated and the code
returned to the caller of :c:func:`PoolInit()`` or :c:func:`PoolCreate()`. Note that
the :c:type:`PoolStruct` isn't made fully valid until :c:func:`PoolInit()` returns.
The ``finish`` field is the class's finish method. This method is
called via the generic function :c:func:`PoolFinish()`, which is in turn
called by :c:func:`PoolDestroy()`. It is expected to finalise the pool
instance structure and release any resources allocated to the pool, it
is expected to release the memory associated with the pool instance
structure. Note that the pool is valid when it is passed to
``finish``. The :c:type:`PoolStruct` (generic part) is finished off when the
class's ``finish`` method returns.
The ``alloc`` field is the class's allocation method. This method is
called via the generic function :c:func:`PoolAlloc()`. It is expected to
return a pointer to a fresh (that is, not overlapping with any other
live object) object of the required size. Failure to allocate should
be indicated by returning an appropriate Error code, and in such a
case, ``*pReturn`` should not be updated. Classes are not required to
provide this method, but they should provide at least one of ``alloc``
and ``bufferCreate``.
.. note::
There is no ``bufferCreate``. Gareth Rees, 2013-04-14.
The ``free_`` field is the class's free method. This is intended
primarily for manual style pools. this method is called via the
generic function :c:func:`PoolFree()`. The parameters to this method are
required to correspond to a previous allocation request (possibly via
a buffer). It is an assertion by the client that the indicated object
is no longer required and the resources associated with it can be
recycled. Pools are not required to provide this method.
The ``bufferInit`` field is the class's buffer initialization method.
It is called by the generic function :c:func:`BufferCreate()`, which allocates
the buffer descriptor and initializes the generic fields. The pool may
optionally adjust these fields or fill in extra values when
``bufferInit`` is called, but often pools set ``bufferInit`` to
:c:func:`PoolTrivBufferInit()` because they don't need to do any. If
``bufferInit`` returns a result code other than ``ResOK``, the buffer
structure is deallocated and the code is returned to the called of
:c:func:`BufferCreate()`. Note that the :c:type:`BufferStruct` isn't fully valid
until :c:func:`BufferCreate()` returns.
The ``bufferFinish`` field is the class's buffer finishing method. It
is called by the the generic function :c:func:`BufferDestroy()`. The pool is
expected to detach the buffer from any memory and prepare the buffer
for destruction. The class is expected to release the resources
associated with the buffer structure, and any unreserved memory in the
buffer may be recycled. It is illegal for a buffer to be destroyed
when there are pending allocations on it (that is, an allocation has
been reserved, but not committed) and this is checked in the generic
function. This method should be provided if and only if
``bufferCreate`` is provided. [there is no ``bufferCreate`` -- drj
1997-08-19]
The ``condemn`` field is used to condemn a pool. This method is called
via the generic function :c:func:`PoolCondemn()`. The class is expected to
condemn a subset (possible the whole set) of objects it manages and
participate in a global trace to determine liveness. The class should
register the refsig of the condemned set with the trace using
:c:func:`TraceCondemn()`. The class should expect fix requests (via the fix
method below) during a global trace. Classes are not required to
provide this method, but it is expected that automatic style classes
will. This interface is expected to change in the future.
.. note::
``condemn`` now takes an action and a segment and should condemn
the segment (turn it white) if it corresponds to the
interpretation of the action. David Jones, 1997-08-19.
It is now called ``whiten``. David Jones, 1998-02-02.
The ``mark`` field is used to mark an entire pool. This method is
called via the generic function :c:func:`PoolMark()`. The class should
consider all of its objects, except any set that has been condemned in
this trace, to be marked, that is ready for scanning. The class should
arrange that any appropriate invariants are preserved possibly by the
Protection interface. Classes are not required to provide this method,
and not doing so indicates that all instances of this class will have
no fixable or traceable references in them.
.. note::
``mark`` is no longer present: ``grey`` turns an entire segment
grey. David Jones, 1997-08-19.
The ``scan`` field is used to perform scanning. This method is called
via the generic function :c:func:`PoolScan()`. The class should scan the
segment specified. It should scan all the known live (marked, that is,
those objects on which fix has been called) on the segment and
accumulate a summary of *all* the objects on the segment. This means
that mark and sweep pools may have to jump through hoops a little bit
(see design.mps.poolasm.summary for a pedagogical example). Classes
are not required to provide this method, and not doing so indicates
that all instances of this class will have no fixable or traceable
reference in them.
.. note::
The ``scan`` method now takes an extra return parameter which
classes should use to indicate whether they scanned all objects in
segment or not. Classes should return summary only of object they
scanned. Caller of this method (:c:func:`TraceScan()`) is responsible
for updating summaries correctly when not a total scan. Hence no
jumping through hoops required. David Jones, 1998-01-30.
The ``fix`` field is used to perform fixing. This method is called via
the generic function :c:func:`TraceFix()`. It indicates that the specified
reference has been found and the class should consider the object
live. There is provision for adjusting the value of the reference (to
allow for classes that move objects). Classes are not required to
provide this method, and not doing so indicates that the class is not
automatic style (ie it does not use global tracing to determine
liveness).
The ``reclaim`` field is used to reclaim memory. This method is called
via the generic function :c:func:`PoolReclaim()`. It indicates that the trace
has fixed all references to reachable objects.
.. note::
Actually it indicates that any remaining white objects have now
been proved unreachable, hence are dead. David Jones, 1997-08-19.
The class should consider objects that have been condemned and not
fixed in this trace to be dead and may reclaim the resources
associated with them. Classes are not required to provide this method.
.. note::
``reclaim`` is now called on each segment. David Jones,
1997-08-19.
The ``access`` field is used to indicate client access. This method is
called via the generic functions :c:func:`SpaceAccess()` and
:c:func:`PoolAccess()`. It indicates that the client has attempted to access
the specified region, but has been denied and the request trapped due
to a protection state. The class should perform any work necessary to
remove the protection whilst still preserving appropriate invariants
(typically this will be scanning work). Classes are not required to
provide this method, and not doing so indicates they never protect any
memory managed by the pool.
.. note::
``access`` is no longer present. David Jones, 1997-08-19.
:mps:tag:`method.act` ``act`` is called when the MPM has decided to execute
an action that the class declared. The Class should arrange execution
of the associated work (usually by beginning an incremental trace).
:mps:tag:`method.walk` ``walk`` is used by the heap walker. ``walk`` is only
required to be implemented by classes which specify the AttrFMT
attribute (formatted pools). The ``walk`` method should apply the
passed in function (along with its closure variables (which are also
passed in) and the object format) to all *black* objects in the
segment. Padding objects may or may not be included in the walk at the
classes discretion, in any case in will be the responsibility of the
client to do something sensible with padding objects.
.. note::
What about broken hearts? David Jones, 1998-01-30.
The ``describe`` field is used to print out a description of a pool.
This method is called via the generic function :c:func:`PoolDescribe()`. The
class should emit an textual description of the pool's contents onto
the specified stream. Each line should begin with two spaces. Classes
are not required to provide this method.
Events
------
:mps:tag:`replay` To work with the allocation replayer (see
design.mps.telemetry.replayer), the pool has to emit an event for each
call to an external interface, containing all the parameters passed by
the user. If a new event type is required to carry this information,
the replayer (impl.c.eventrep) must then be extended to recreate the
call.
:mps:tag:`replay.Init` In particular, the ``init`` method should emit a
``PoolInit<foo>`` event with all the pool parameters.
Text
-----
:mps:tag:`alloc.size` The pool class implementation defines the meaning of
the "size" parameter to the ``alloc`` and ``free`` methods. It may not
actually correspond to a number of bytes of memory.
:mps:tag:`alloc.size.align` In particular, the class may allow an unaligned
size to be passed.

388
mps/manual/html/_sources/design/collection.txt
View file

@ -1,6 +1,390 @@
.. _design-collection:
.. index::
pair: collection framework; design
.. _design-collection:
.. include:: ../../converted/collection.rst
Collection framework
====================
.. mps:prefix:: design.mps.collection
Introduction
------------
:mps:tag:`intro` This document describes the Collection Framework. It's a
framework for implementing garbage collection techniques and
integrating them into a system of collectors that all cooperate in
recycling garbage.
Overview
--------
:mps:tag:`framework` MPS provides a framework that allows the integration of
many different types of GC strategies and provides many of the basic
services that those strategies use.
:mps:tag:`framework.cover` The framework subsumes most major GC strategies
and allows many efficient techniques, like in-line allocation or
software barriers.
:mps:tag:`framework.overhead` The overhead due to cooperation is low.
.. note::
But not non-existent. Can we say something useful about it?
:mps:tag:`framework.benefits` The ability to combine collectors contributes
significantly to the flexibility of the system. The reduction in code
duplication contributes to reliability and integrity. The services of
the framework make it easier to write new MM strategies and
collectors.
:mps:tag:`framework.mpm` The Collection Framework is merely a part of the
structure of the MPM. See design.mps.architecture and design.mps.arch
for the big picture.
.. note::
Those two documents should be combined into one. Pekka P. Pirinen,
1998-01-15.
Other notable components that the MPM manages to integrate into a
single framework are manually-managed memory and finalization services
(see design.mps.finalize).
.. note::
A document describing the design of manually-managed memory is
missing. Pekka P. Pirinen, 1998-01-15.
:mps:tag:`see-also` This document assumes basic familiarity with the ideas
of pool (see design.mps.arch.pools) and segment (see
design.mps.seg.over).
Collection abstractions
-----------------------
Colours, scanning and fixing
............................
:mps:tag:`state` The framework knows about the three colours of the
tri-state abstraction and free blocks. Recording the state of each
object is the responsibility of the pool, but the framework gets told
about changes in the states and keeps track of colours in each
segment. Specifically, it records whether a segment might contain
white, grey and black objects with respect to each active trace (see
:mps:ref:`.tracer`)
.. note::
Black not currently implemented. Pekka P. Pirinen, 1998-01-04.
(A segment might contain objects of all colours at once, or none.)
This information is approximate, because when an object changes
colour, or dies, it usually is too expensive to determine if it was
the last object of its former colour.
:mps:tag:`state.transitions` The possible state transitions are as follows::
free ---alloc--> black (or grey) or white or none
none --condemn-> white
none --refine--> grey
grey ---scan---> black
white ----fix---> grey (or black)
black --revert--> grey
white --reclaim-> free
black --reclaim-> none
:mps:tag:`none-is-black` Outside of a trace, objects don't really have
colour, but technically, the colour is black. Objects are only
allocated grey or white during a trace, and by the time the trace has
finished, they are either dead or black, like the other surviving
objects. We might then reuse the colour field for another trace, so
it's convenient to set the colour to black when allocating outside a
trace. This means that refining the foundation
(analysis.tracer.phase.condemn.refine), actually turns black segments
grey, rather than vice versa, but the principle is the same.
:mps:tag:`scan-fix` "Scanning" an object means applying the "fix" function
to all references in that object. Fixing is the generic name for the
operation that takes a reference to a white object and makes it
non-white (usually grey, but black is a possibility, and so is
changing the reference as we do for weak references). Typical examples
of fix methods are copying the object into to-space or setting its
mark bit.
:mps:tag:`cooperation` The separation of scanning and fixing is what allows
different GC techniques to cooperate. The scanning is done by a method
on the pool that the scanned object resides in, and the fixing is done
by a method on the pool that the reference points to.
:mps:tag:`scan-all` Pools provide a method to scan all the grey objects in a
segment.
Reference sets
..............
:mps:tag:`refsets` The cost of scanning can be significantly reduced by
storing remembered sets. We have chosen a very compact and efficient
implementation, called reference sets, or refsets for short (see
idea.remember).
.. note::
design.mps.refset is empty! Perhaps some of this should go there.
Pekka P. Pirinen, 1998-02-19.
This makes the cost of maintaining them low, so we maintain them for
all references out of all scannable segments.
:mps:tag:`refsets.approx` You might describe refsets as summaries of all
references out of an area of memory, so they are only approximations
of remembered sets. When a refset indicates that an interesting
reference might be present in a segment, we still have to scan the
segment to find it.
:mps:tag:`refsets.scan` The refset information is collected during scanning.
The scan state protocol provides a way for the pool and the format
scan methods to cooperate in this, and to pass this information to the
tracer module which checks it and updates the segment (see
design.mps.scan).
.. note::
Actually, there's very little doc there. Pekka P. Pirinen,
1998-02-17.
:mps:tag:`refsets.maintain` The MPS tries to maintain the refset information
when it moves or changes object.
:mps:tag:`refsets.pollution` Ambiguous references and pointers outside the
arena will introduce spurious zones into the refsets. We put up with
this to keep the scanning costs down. Consistency checks on refsets
have to take this into account.
:mps:tag:`refsets.write-barrier` A write-barrier are needed to keep the
mutator from invalidating the refsets when writing to a segment. We
need one on any scannable segment whose refset is not a superset of
the mutator's (and that the mutator can see). If we know what the
mutator is writing and whether it's a reference, we can just add that
reference to the refset (figuring out whether anything can be removed
from the refset is too expensive). If we don't know or if we cannot
afford to keep the barrier up, the framework can union the mutator's
refset to the segment's refset.
:mps:tag:`refset.mutator` The mutator's refset could be computed during root
scanning in the usual way, and then kept up to date by using a
read-barrier. It's not a problem that the mutator can create new
pointers out of nothing behind the read-barrier, as they won't be real
references. However, this is probably not cost-effective, since it
would cause lots of barrier hits. We'd need a read-barrier on every
scannable segment whose refset is not a subset of the mutator's (and
that the mutator can see). So instead we approximate the mutator's
refset with the universal refset.
The tracer
----------
:mps:tag:`tracer` The tracer is an engine for implementing multiple garbage
collection processes. Each process (called a "trace") proceeds
independently of the others through five phases as described in
analysis.tracer. The following sections describe how the action of
each phase fits into the framework. See design.mps.trace for details
.. note::
No, there's not much there, either. Possibly some of this section
should go there. Pekka P. Pirinen, 1998-02-18.
:mps:tag:`combine` The tracer can also combine several traces for some
actions, like scanning a segment or a root. The methods the tracer
calls to do the work get an argument that tells them which traces they
are expected to act for.
.. note::
Extend this.
:mps:tag:`trace.begin` Traces are started by external request, usually from
a client function or an action (see design.mps.action).
:mps:tag:`trace.progress` The tracer gets time slices from the arena to work
on a given trace.
.. note::
This is just a provisional arrangement, in lieu of real progress
control. Pekka P. Pirinen, 1998-02-18.
In each slice, it selects a small amount of work to do, based on the
state of the trace, and does it, using facilities provided by the
pools. .trace.scan: A typical unit of work is to scan a single
segment. The tracer can choose to do this for multiple traces at once,
provided the segment is grey for more than one trace.
:mps:tag:`trace.barrier` Barrier hits might also cause a need to scan :mps:a
segment (see :mps:ref:`.hw-barriers.hit`). Again, the tracer can
:mps:choose to combine traces, when it does this.
:mps:tag:`mutator-colour` The framework keeps track of the colour of the
mutator separately for each trace.
The condemn phase
.................
:mps:tag:`phase.condemn` The agent that creates the trace (see
:mps:ref:`.trace.begin`) determines the condemned set and colours it white.
The tracer then examines the refsets on all scannable segments, and if
it can deduce some segment cannot refer to the white set, it's
immediately coloured black, otherwise the pool is asked to grey any
objects in the segment that might need to be scanned (in copying
pools, this is typically the whole segment).
:mps:tag:`phase.condemn.zones` To get the maximum benefit from the refsets,
we try to arrange that the zones are a minimal superset (for example,
generations uniquely occupy zones) and a maximal subset (there's
nothing else in the zone) of the condemned set. This needs to be
arranged at allocation time (or when copying during collection, which
is much like allocation)
.. note::
Soon, this will be handled by segment loci, see design.mps.locus.
:mps:tag:`phase.condemn.mutator` At this point, the mutator might reference
any objects, that is, it is grey. Allocation can be in any colour,
most commonly white.
.. note::
More could be said about this.
The grey mutator phase
......................
:mps:tag:`phase.grey-mutator` Grey segments are chosen according to some
sort of progress control and scanned by the pool to make them black.
Eventually, the tracer will decide to flip or it runs out of grey
segments, and proceeds to the next phase.
.. note::
Currently, this phase has not been implemented; all traces flip
immediately after condemn. Pekka P. Pirinen, 1998-02-18.
:mps:tag:`phase.grey-mutator.copy` At this stage, we don't want to copy
condemned objects, because we would need an additional barrier to keep
the mutator's view of the heap consistent (see
analysis.async-gc.copied.pointers-and-new-copy).
:mps:tag:`phase.grey-mutator.ambig` This is a good time to get all ambiguous
scanning out of the way, because we usually can't do any after the
flip and because it doesn't cause any copying.
.. note::
Write a detailed explanation of this some day.
The flip phase
..............
:mps:tag:`phase.flip` The roots (see design.mps.root) are scanned. This has
to be an atomic action as far as the mutator is concerned, so all
threads are suspended for the duration.
:mps:tag:`phase.flip.mutator` After this, the mutator is black: if we use a
strong barrier (analysis.async-gc.strong), this means it cannot refer
to white objects. Allocation will be in black (could be grey as well,
but there's no point to it).
The black mutator phase
.......................
:mps:tag:`phase.black-mutator` Grey segments are chosen according to some
sort of progress control and scanned by the pool to make them black.
Eventually, the tracer runs out of segments that are grey for this
trace, and proceeds to the next phase.
:mps:tag:`phase.black-mutator.copy` At this stage white objects can be
relocated, because the mutator cannot see them (as long as a strong
barrier is used, as we must do for a copying collection, see
analysis.async-gc.copied.pointers).
The reclaim phase
.................
:mps:tag:`phase.reclaim` The tracer finds the remaining white segments and
asks the pool to reclaim any white objects in them.
:mps:tag:`phase.reclaim.barrier` Once a trace has started reclaiming
objects, the others shouldn't try to scan any objects that are white
for it, because they might have dangling pointers in them.
.. note::
Needs cross-reference to document that is yet to be written.
Currently, we reclaim atomically, but it could be incremental, or
even overlapped with a new trace on the same condemned set.
Pekka P. Pirinen, 1997-12-31.
Barriers
--------
.. note::
An introduction and a discussion of general principles should go
here. This is a completely undesigned area.
Hardware barriers
.................
:mps:tag:`hw-barriers` Hardware barrier services cannot, by their very
nature, be independently provided to each trace. A segment is either
protected or not, and we have to set the protection on a segment if
any trace needs a hardware barrier on it.
:mps:tag:`hw-barriers.supported` The framework currently supports
segment-oriented Appel-Ellis-Li barriers
(analysis.async-gc.barrier.appel-ellis-li), and write-barriers for
keeping the refsets up-to-date. It would not be hard to add Steele
barriers (analysis.async-gc.barrier.steele.scalable).
:mps:tag:`hw-barriers.hit` When a barrier hit happens, the arena determines
which segment it was on. The segment colour info is used to determine
whether it had trace barriers on it, and if so, the appropriate
barrier action is performed, using the methods of the owning pool. If
the segment was write-protected, its refset is unioned with the refset
of the mutator.
.. note:: In practice this is ``RefSetUNIV``.
:mps:tag:`hw-barriers.hit.multiple` Fortunately, if we get a barrier hit on
a segment with multiple trace barriers on it, we can scan it for all
the traces that it had a barrier for.
.. note:: Needs link to unwritten section under :mps:ref:`.combine`.
Software barriers
.................
.. note::
Write something about software barriers.

540
mps/manual/html/_sources/design/config.txt
View file

@ -1,6 +1,542 @@
.. _design-config:
.. index::
pair: configuration; design
.. _design-config:
.. include:: ../../converted/config.rst
MPS Configuration
=================
.. mps:prefix:: design.mps.config
Introduction
------------
:mps:tag:`intro` This document describes how the `Memory Pool System
<http://www.ravenbrook.com/project/mps/>`_ source code is configured so
that it can target different architectures, operating systems, build
environments, varieties, and products.
Requirements
------------
:mps:tag:`req.import` The MPS must be simple to include in third-party projects.
:mps:tag:`req.arch` Allow architecture specific configurations of the MPS, so
that we can vary the MPS according to the target architecture.
:mps:tag:`req.os` Allow operating system specific configurations of the MPS,
so that we can vary the MPS according to the target OS.
:mps:tag:`req.builder` Allow build environment specific configurations of the
MPS, so that we can vary the MPS according to the compiler, etc.
:mps:tag:`req.var` Allow configurations with different amounts of
instrumentation (assertions, metering, etc.).
:mps:tag:`req.impact` The configuration system should have a minimal effect on
maintainability of the implementation.
:mps:tag:`req.port` The system should be easy to port across platforms.
:mps:tag:`req.maint` Maintenance of the configuration and build system should
not consume much developer time.
Retired requirements
....................
:mps:tag:`req.prod` Allow product specific configurations of the MPS, so that
we can build variants of the MPS for use in different products. This
requirement has been retired on 2012-09-03 as part of work on the
variety-reform_ branch. Client-specific customisation of the MPS will
be handled in source control, while the MPS source remains generic, to
reduce costs and increase reliability. See [RB_2012-09-13]_.
.. _variety-reform: /project/mps/branch/2012-08-15/variety-reform
Definitions
-----------
:mps:tag:`def.platform` A *platform* is a combination of an architecture
(:mps:ref:`.def.arch`), an operating system (:mps:ref:`.def.os`), and a builder
(:mps:ref:`.def.builder`). The set of supported platforms is maintained in the
`Platforms section of "Building the Memory Pool System"
<../manual/html/guide/build.html#platforms>`_.
:mps:tag:`def.arch` An *architecture* is processor type with associated calling
conventions and other binary interface stuff these days often called the
`ABI <http://en.wikipedia.org/wiki/Application_binary_interface>`_.
Most importantly for the MPS it determines the layout of the register
file, thread context, and thread stack.
:mps:tag:`def.os` An *operating system* is the interface to external resources.
Most importantly for the MPS it determines the low level interface to
virtual memory (if any) and threading.
:mps:tag:`def.builder` A *builder* is the tools (C compiler, etc.) used to make
the target (:mps:ref:`.def.target`). The MPS minimises use of compiler-specific
extensions, but this is handy for suppressing warnings, inlining hints,
etc.
:mps:tag:`def.var` A *variety* determines things like the amount of debugging,
internal consistency checking, annotation, etc. In modern IDEs this
called a "build configuration" and the usual default is to have two:
"debug" and "release". The MPS predates this convention, but the concept
is the same.
:mps:tag:`def.prod` A *product* is the intended product into which the MPS will
fit, e.g. ScriptWorks, Dylan, etc. We no longer maintain this concept
as a dimension of configuration since :mps:ref:`.req.prod` has been retired.
:mps:tag:`def.target` The *target* is the result of the build.
Overview
--------
:mps:tag:`import.source` The MPS can be simply included in client products as
source code. Since `version 1.110`_ we made it possible to simply
include the file ``mps.c`` in a client's build process, without
requiring a separate build of the MPS or linking a library. This is
described `section 2.3.1, "Compiling for production" of the MPS manual
<../manual/html/guide/build.html#compiling-for-production>`_.
.. _`version 1.110`: http://www.ravenbrook.com/project/mps/version/1.110/
:mps:tag:`no-gen` No generated code or external tools are required. On most
platforms the only tool is the C compiler. On 64-bit Windows we require
the assembler since Microsoft withdrew in-line assembler from their C
compiler.
:mps:tag:`no-spaghetti` Several of the MPS team have worked on some extremely
messy code bases which used a great number of ``#ifdef`` statements.
These quickly became very expensive to maintain and develop. The
general rule in the MPS is "no ``#ifdefs``". Instead, platform-specific
code is kept in separate source files and selected by carefully controlled
``#ifdefs``, such as in `mps.c <../code/mps.c>`_.
:mps:tag:`min-dep` Dependency on a particular configuration should be
minimized and localized when developing code. This is enshrined in the
general rules for implementation [ref?] that are enforced by MPS
development procedures including code review and inspection.
The build system
----------------
Abstract build function
.......................
:mps:tag:`build.fun` The MPS implementation assumes only a simple "build
function" that takes a set of sources, possibly in several languages,
compiles them with a set of predefined preprocessor symbols, and links
the result with a set of libraries to form the target::
target := build(<defs>, <srcs>, <libs>)
:mps:tag:`build.sep` Separate compilation and linkage can be seen as a
memoization of this function, and is not strictly necessary for the
build. Indeed, since `version 1.110` we found that modern compilers
are quite happy to compile the whole MPS in one go :mps:ref:`.import.source`.
:mps:tag:`build.cc` A consequence of this approach is that it should always
be possible to build a complete target with a single UNIX command line
calling the compiler driver (usually "cc" or "gcc"), for example::
cc -o main -DCONFIG_VAR_DF foo.c bar.c baz.s -lz
:mps:tag:`build.defs` The "defs" are the set of preprocessor macros which are to be
predefined when compiling the module sources::
CONFIG_VAR_<variety-code>
The variety-codes are the letter code that appears after "variety." in
the tag of the relevant variety document (see variety.*) converted to
upper case. Currently (2012-09-03):
:mps:tag:`var.hot` :c:macro:`HOT`
Intended for release in products. Optimised, reduced internal
checking, especially on the critical path [RB_2012-09-07]_.
:mps:tag:`var.cool` :c:macro:`COOL`
Intended for use during development. Moderately thorough internal
consistency checking. Reduced optimisation to allow for
single-stepping.
:mps:tag:`var.rash` :c:macro:`RASH`
No internal checking at all. Slight performance improvement over
:mps:ref:`.var.hot` at the cost of early detection of memory management
bugs. We do not advise use of this variety, as memory management
bugs tend to be extremely expensive to deal with.
:mps:tag:`var.diag` :c:macro:`DIAG` (deprecated)
This variety does some client-specific analysis and produces some
specialised diagnostic output, and is not intended for general use.
It will be phased out of the open sources.
:mps:tag:`default.hot` If no :c:macro:`CONFIG_VAR` is present, :c:macro:`HOT` is assumed in
`config.h`_.
:mps:tag:`build.srcs` The "srcs" are the set of sources that must be
compiled in order to build the target. The set of sources may vary
depending on the configuration. For example, different sets of sources
may be required to build different architectures.
.. note::
This is a dependency between the makefile (or whatever) and the
module configuration in `config.h`_.
:mps:tag:`build.libs` The "libs" are the set of libraries to which the
compiled sources must be linked in order to build the target. For
example, when building a test program, it might include the ANSI C
library and an operating system interface library.
File Structure
..............
:mps:tag:`file.dir` The MPS source code is arranged in a single directory
called "code" containing all the sources for the whole family of
targets.
:mps:tag:`file.base` The names of sources must be unique in the first eight
characters in order to conform to FAT filesystem naming restrictions.
(Do not scoff -- this has been an important requirement as recently as
2012!)
:mps:tag:`file.ext` The extension may be up to three characters and directly
indicates the source language.
:mps:tag:`file.platform` Platform-specific files include the platform code
in their name. See :mps:ref:`.mod.impls`.
Modules and naming
..................
:mps:tag:`mod.unique` Each module has an identifier which is unique within the MPS.
:mps:tag:`mod.impls` Each module has one or more implementations which may be
in any language supported by the relevant build environment.
:mps:tag:`mod.primary` The primary implementation of a module is written in
target-independent ANSI C in a source file with the same name as the
module.
:mps:tag:`mod.an` Where there are platform-specific implementations and an
inferior portable ANSI C fallback implementation, "an" is used in
place of the platform code.
:mps:tag:`mod.secondary` The names of other implementations should begin
with the same prefix (the module id or a shortened version of it) and
be suffixed with on or more target parameter codes (defined below). In
particular, the names of assembly language sources must include the
target parameter code for the relevant architecture.
:mps:tag:`mod.example` For example, the stack scanner is defined in `ss.h
<../code/ss.h>`_ (which is platform-independent). It has some
platform-independent C in `ss.c <../code/ss.c>`_ and, for example,
`ssw3i6mv.c <../code/ssw3i6mv.c>`_ is specific to Windows on the x64
architecture built with Microsoft Visual C.
Build system rationale
......................
:mps:tag:`build.rat` This simple design makes it possible to build the MPS
using many different tools. Microsoft Visual C and other graphical
development tools do not support much in the way of generated sources,
staged building, or other such stuff. The Visual C and Xcode "project"
files correspond closely to a closure of the build function
(:mps:ref:`.build.fun`). The simplicity of the build function has also made it
easy to set up builds using NMAKE (DOS), MPW (Macintosh), and to get the
MPS up and running on other platforms such as FreeBSD and Linux in very
little time. The cost of maintaining the build systems on these various
platforms is also reduced to a minimum, allowing the MPS developers to
concentrate on primary development. The source code is kept simple and
straightforward. When looking at MPS sources you can tell exactly what
is going to be generated with very little context. The sources are not
munged beyond the standard ANSI C preprocessor.
:mps:tag:`build.port` The portability requirement (:mps:ref:`.req.port`) implies that
the build system must use only standard tools that will be available on
all conceivable target platforms. Experience of development
environments on the Macintosh (Metrowerks Codewarrior) and Windows NT
(Visual C++) indicates that we cannot assume much sophistication in the
use of file structure by development environments. The best that we can
hope for is the ability to combine a fixed list of source files,
libraries, and predefined preprocessor symbols into a single target.
:mps:tag:`build.maint` The maintainability requirement (:mps:ref:`.req.maint`) implies
that we don't spend time trying to develop a set of tools to support
anything more complicated than the simple build function described
above. The effort in constructing and maintaining a portable system of
this kind is considerable. Such efforts failed in the Electronic
Publishing division of Harlequin.
Implementation
--------------
:mps:tag:`impl` The two implementation files `config.h`_ and `mpstd.h`_ can be
seen as preprocessor programs which "accept" build parameters and "emit"
configuration parameters (:mps:ref:`.fig.impl`). The build parameters are
defined either by the builder (in the case of target detection) or by
the build function (in the case of selecting the variety).
:mps:tag:`fig.impl`
=========================== ============== ===========================================
Build parameters Source file Configuration parameters
=========================== ============== ===========================================
:c:macro:`CONFIG_VAR_HOT` ⟶ ``config.h`` ⟶ :c:macro:`MPS_ASSERT_STRING`, etc.
``_WIN32`` ⟶ ``mpstd.h`` ⟶ :c:macro:`MPS_OS_W3`, etc.
=========================== ============== ===========================================
:mps:tag:`impl.dep` No source code, other than the directives in `config.h`_
and `mpstd.h`_, should depend on any build parameters. That is,
identifers beginning "CONFIG\_" should only appear in impl.h.config.
Code may depend on configuration parameters in certain, limited ways, as
defined below (:mps:ref:`.conf`).
.. _`config.h`: <../code/config.h>
.. _`mpstd.h`: <../code/mpstd.h>
Target platform detection
.........................
:mps:tag:`pf` The target platform is "detected" by the preprocessor directives in
`mpstd.h`_.
:mps:tag:`pf.form` This file consists of sets of directives of the form::
#elif <conjunction of builder predefinitions>
#define MPS_PF_<platform code>
#define MPS_OS_<operating system code>
#define MPS_ARCH_<architecture code>
#define MPS_BUILD_<builder code>
#define MPS_T_WORD <word type>
#define MPS_T_ULONGEST <longest unsigned integer type>
#define MPS_WORD_SHIFT <word shift>
#define MPS_PF_ALIGN <minimum alignment>
:mps:tag:`pf.detect` The conjunction of builder predefinitions is a constant
expression which detects the target platform. It is a logical AND of
expressions which look for preprocessor symbols defined by the build
environment to indicate the target. These must be accompanied by a
reference to the build tool documentation from which the symbols came.
For example::
/* "Predefined Macros" from "Visual Studio 2010" on MSDN
* <http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.100).aspx>. */
#elif defined(_MSC_VER) && defined(_WIN32) && defined(_M_IX86)
:mps:tag:`pf.codes` The declarations of the platform, operating system,
architecture, and builder codes define preprocessor macros corresponding
the the target detected (:mps:ref:`.pf.detect`). For example::
#define MPS_PF_W3I3MV
#define MPS_OS_W3
#define MPS_ARCH_I3
#define MPS_BUILD_MV
:mps:tag:`pf.word` The declaration of :c:macro:`MPS_T_WORD` defines the unsigned
integral type which corresponds, on the detected target, to the
machine word. It is used to defined the MPS Word type
(design.mps.type.word). For example::
#define MPS_T_WORD unsigned long
We avoid using ``typedef`` here because `mpstd.h`_ could potentially
be included in assembly language source code.
:mps:tag:`pf.word-width` The declaration of :c:macro:`MPS_WORD_WIDTH` defines the
number of bits in the type defined by :c:macro:`MPS_T_WORD` (:mps:ref:`.pf.word`) on the
target. For example::
#define MPS_WORD_WIDTH 32
:mps:tag:`pf.word-shift` The declaration of :c:macro:`MPS_WORD_SHIFT` defines the log
to the base 2 of :c:macro:`MPS_WORD_WIDTH`. For example::
#define MPS_WORD_SHIFT 5
:mps:tag:`pf.pf-align` The declaration of :c:macro:`MPS_PF_ALIGN` defines the minimum
alignment which must be used for a memory block to permit any normal
processor memory access. In other words, it is the maximum alignment
required by the processor for normal memory access. For example::
#define MPS_PF_ALIGN 4
:mps:tag:`pf.ulongest` The declaration of :c:macro:`MPS_T_ULONGEST` defines the
longest available unsigned integer type on the platform. This is
usually just ``unsigned long`` but under Microsoft C on 64-bit Windows
``unsigned long`` is just 32-bits (curse them!) For example::
#define MPS_T_ULONGEST unsigned __int64
:mps:tag:`pf.pf-string` The declaration of :c:macro:`MPS_PF_STRING` defines a string
that is used to identify the target platform in `version.c
<../code/version.c>`_. For example::
#define MPS_PF_STRING "w3i6mv"
Target varieties
................
:mps:tag:`var` The target variety is handled by preprocessor directives in
impl.h.config.
:mps:tag:`var.form` The file contains sets of directives of the form::
#if defined(CONFIG_VAR_COOL)
#define CONFIG_ASSERT
#define CONFIG_ASSERT_ALL
#define CONFIG_STATS
:mps:tag:`var.detect` The configured variety is one of the variety
preprocessor definitions passed to the build function
(:mps:ref:`.build.defs`), for example, :c:macro:`CONFIG_VAR_COOL`. These are
decoupled in order to keep the number of supported varieties small,
controlling each feature (for example, assertions) by a single
preprocessor definition, and maintaining flexibility about which
features are enabled in each variety.
:mps:tag:`var.symbols` The directives should define whatever symbols are
necessary to control featrues. These symbols parameterize other parts
of the code, such as the declaration of assertions, etc. The symbols
should all begin with the prefix :c:macro:`CONFIG_`.
Source code configuration
-------------------------
:mps:tag:`conf` This section describes how the configuration may affect the
source code of the MPS.
:mps:tag:`conf.limit` The form of dependency allowed is carefully limited to
ensure that code remains maintainable and portable (:mps:ref:`.req.impact`).
:mps:tag:`conf.min` The dependency of code on configuration parameters should
be kept to a minimum in order to keep the system maintainable
(:mps:ref:`.req.impact`).
Configuration Parameters
........................
:mps:tag:`conf.params` The compilation of a module is parameterized by::
MPS_ARCH_<arch-code>
MPS_OS_<os-code>
MPS_BUILDER_<builder-code>
MPS_PF_<platform-code>
Abstract and Concrete Module Interfaces
.......................................
:mps:tag:`abs.caller` Basic principle: the caller musn't be affected by
configuration of a module. This reduces complexity and dependency of
configuration. All callers use the same abstract interface. Caller
code does not change.
:mps:tag:`abs.interface` Abstract interface includes:
- method definitions (logical function prototypes which may be macro methods)
- names of types
- names of constants
- names of structures and fields which form part of the interface, and
possibly their types, depending on the protocol defined
- the protocols
:mps:tag:`abs.rule` The abstract interface to a module may not be altered by a
configuration parameter. However, the concrete interface may vary.
For example, this isn't allowed, because there is a change in the interface::
#if defined(PROT_FOO)
void ProtSpong(Foo foo, Bar bar);
#else
int ProtSpong(Bar bar, Foo foo);
#endif
This example shows how::
#ifdef PROTECTION
void ProtSync(Space space);
/* more decls. */
#else /* PROTECTION not */
#define ProtSync(space) NOOP
/* more decls. */
#endif /* PROTECTION */
or::
#if defined(PROT_FOO)
typedef struct ProtStruct {
int foo;
} ProtStruct;
#define ProtSpong(prot) X((prot)->foo)
#elif defined(PROT_BAR)
typedef struct ProtStruct {
float bar;
} ProtStruct;
#define ProtSpong(prot) Y((prot)->bar)
#else
#error "No PROT_* configured."
#endif
Configuration parameters may not be used to vary implementations in C files.
For example, this sort of thing::
int map(void *base, size_t size)
{
#if defined(MPS_OS_W3)
VirtualAlloc(foo, bar, base, size);
#elif defined(MPS_OS_SU)
mmap(base, size, frob);
#else
#error "No implementation of map."
#endif
}
This violates :mps:ref:`.no-spaghetti`.
To document
-----------
- What about constants in config.h?
- Update files to refer to this design document.
- Explain the role of ``mps.c``
- Reference to ``build.txt``
- Procedures for adding an architecture, etc.
- Reduce duplication in this document (especially after
`Configuration Parameters`_ which looks like it's been pasted in from
elsewhere.)
References
----------
.. [RB_2012-09-07] Richard Brooksby. Ravenbrook Limited. 2012-09-07. "`The critical path through the MPS <http://www.ravenbrook.com/project/mps/master/design/critical-path>`__".
.. [RB_2012-09-13] Richard Brooksby. Ravenbrook Limited. 2013-09-13. "`The Configura CET custom mainline <https://info.ravenbrook.com/mail/2012/09/13/16-43-35/0/>`__".
.. [PP_2005-03-01] Pekka Pirinen. Global Graphics. 2005-03-01. "`MPS platforms <https://info.ravenbrook.com/mail/2005/03/01/15-45-17/0/>`__".

360
mps/manual/html/_sources/design/critical-path.txt
View file

@ -1,8 +1,364 @@
.. _design-critical-path:
.. index::
single: critical path
single: path; critical
single: Memory Pool System; critical path
.. _design-critical-path:
.. include:: ../../converted/critical-path.rst
The critical path through the MPS
=================================
single: critical path
single: path; critical
single: Memory Pool System; critical path
Introduction
------------
The critical path is a key concept in the design of the `Memory Pool
System <http://www.ravenbrook.com/project/mps/>`_. Code on the critical
path is usually executed more than any other code in the process. A
change of just one instruction on the critical path can make as much as
a 1% difference in overall run-time. A lot of the design of the MPS is
arranged around making the critical path as short and fast as possible.
This document describes the critical path and explains some of that
design, with reference to more detailed documents.
What makes the critical path critical
-------------------------------------
In order to determine which object can be recycled, the garbage
collector has to frequently examine a very large number of pointers in
the program's objects. It does this by scanning_ memory, both
allocated objects and roots (such as the thread stacks).
This means that the scanning functions must loop over pretty much *every
word in memory* sooner or later. The MPS takes great pains to avoid
scanning memory which does not need scanning, but to get good
performance, scanning must be highly optimised.
What's more, the scanning functions apply an operation called "fix" to
every pointer (or potential pointer) that they find in the objects in
memory. Fixing also attempts to eliminate uninteresting pointers as
fast as possible, but it has to do some work on every object that is
being considered for recycling, and that can be a large proportion of
the objects in existence. The path through fixing must also be highly
optimised, especially in the early stages.
How the MPS avoids scanning and fixing
--------------------------------------
This is just a brief overview of how the MPS is designed to reduce
unnecessary scanning and fixing.
Firstly, the MPS must occasionally decide which objects to try to
recycle. It does this using various facts it knows about the objects,
primarily their age and whether they've survived previous attempts at
recycling them. It then `"condemns"`_ a large number of objects
at once, and each of these objects must be "preserved" by fixing
references to them.
When the MPS condemns objects it chooses sets of objects in a small set
of "zones" in memory (preferably a single zone). The zone of an object
can be determined extremely quickly from its address, without looking at
the object or any other data structure.
The MPS arranges that objects which will probably die at the same time
are in the same zones.
The MPS allocates in "segments". Each segment is of the order of one
"tract" of memory (generally the same as the operating system page
size, usually 4KiB or 8KiB) but may be larger if there are large
objects inside. The MPS maintains a "summary" of the zones pointed to
by all the pointers in a segment from previous scans.
So, once the MPS has decided what to condemn, it can quickly eliminate
all segments which definitely do not point to anything in those zones.
This avoids a large amount of scanning. It is an implementation of a
`remembered set`_, though it is unlike that in most other GCs.
In addition, the fix operation can quickly ignore pointers to the wrong
zones. This is called the "zone check" and is a BIBOP_ technique.
Even if a pointer passes the zone check, it may still not point to a
segment containing condemned objects. The next stage of the fix
operation is to look up the segment pointed to by the pointer and see if
it was condemned. This is a fast lookup.
After that, each pool class must decide whether the pointer is to a
condemned object and do something to preserve it. This code is still
critical. The MPS will have tried to condemn objects that are dead, but
those objects are still likely to be in segments with other objects that
must be preserved. The pool class fix method must quickly distinguish
between them.
Furthermore, many objects will be preserved at least once in their
lifetime, so even the code that preserves an object needs to be highly
efficient. (Programs in languages like ML might not preserve 95% of
their objects even once, but many other programs will preserve nearly
all of theirs many times.)
Where to find the critical path
-------------------------------
Very briefly, the critical path consists of five stages:
1. The scanner, which iterates over pointers in objects. The MPS has
several internal scanners, but the most important ones will be format
scanners in client code registered through :c:func:`mps_format_create()`
functions.
.. note::
There needs to be a chapter in the manual explaining how to
write a good scanner. Then that could be linked from here.
2. The first-stage fix, which filters out pointers inline in the
scanner. This is implemented in :c:func:`MPS_FIX()` macros in
mps.h_.
.. _mps.h: ../code/mps.h
3. The second-stage fix, which filters out pointers using general
information about segments. This is ``_mps_fix2`` in
`trace.c <../code/trace.c>`_.
4. The third-stage fix, which filters out pointers using pool-specific
information. Implemented in pool class functions called :c:func:`AMCFix()`,
:c:func:`LOFix()`, etc. in pool*.c.
5. Preserving the object, which might entail
- marking_ it to prevent it being recycled; and/or
- copying_ it and updating the original pointer (or just
updating the pointer, if the object has previously been
copied); and/or
- adding it to a queue of objects to be scanned later, if it
contains pointers.
Found in or near the pool class fix functions.
The format scanner
------------------
The critical path starts when a format scan method is called. That is a
call from the MPS to a client function of type :c:type:`mps_fmt_scan_t`
registered with one of the :c:func:`mps_format_create()` functions in mps.h_.
Here is an example of part of a format scanner for scanning contiguous
runs of pointers, from `fmtdy.c <../code/fmtdy.c>`_, the scanner for the `Open Dylan`_
runtime::
static mps_res_t dylan_scan_contig(mps_ss_t mps_ss,
mps_addr_t *base, mps_addr_t *limit)
{
mps_res_t res;
mps_addr_t *p; /* reference cursor */
mps_addr_t r; /* reference to be fixed */
MPS_SCAN_BEGIN(mps_ss) {
p = base;
loop: if(p >= limit) goto out;
r = *p++;
if(((mps_word_t)r&3) != 0) /* pointers tagged with 0 */
goto loop; /* not a pointer */
if(!MPS_FIX1(mps_ss, r)) goto loop;
res = MPS_FIX2(mps_ss, p-1);
if(res == MPS_RES_OK) goto loop;
return res;
out: assert(p == limit);
} MPS_SCAN_END(mps_ss);
return MPS_RES_OK;
}
(To help with understanding optimisation of this code, it's written in
a pseudo-assembler style, with one line roughly corresponding to each
instruction of an idealized intermediate code.)
The MPS C interface provides macros to try to help optimise this code.
The ``mps_ss`` object is a "scan state" and contains data that is used
to eliminate uninteresting pointers now, and record information which
will be used to reduce scanning in future by maintaining the
remembered set.
The macros :c:func:`MPS_SCAN_BEGIN()` and :c:func:`MPS_SCAN_END()` load key data
from the scan state into local variables, and hopefully into processor
registers. This avoids aliasing values that we know won't change when
calls are made to ``_mps_fix2`` later, and so allows the compiler to
keep the scan loop small and avoid unnecessary memory references.
This scanner knows that words not ending in 0b00 aren't pointers to
objects, so it eliminates them straight away. This is a kind of
`reference tag`_ chosen by the client for its object representation.
Next, the pointer is tested using :c:func:`MPS_FIX1()`. This performs fast
tests on the pointer without using any other memory. In particular, it
does the "zone check" described in section 3. If a pointer fails these
tests, it isn't interesting and can be skipped. It is very important
to proceed to the next pointer as fast as possible in this case.
Having passed these tests, we need to fix the pointer using other data
in memory, and possibly call the MPS to preserve the object. This is
what :c:func:`MPS_FIX2()` does. The important distinction here is that
:c:func:`MPS_FIX2()` can fail and return an error code, which must be
propagated without ado by returning from the scanner. Separating
:c:func:`MPS_FIX1()` from :c:func:`MPS_FIX2()` helps keep the error handling code
away from the tight loop with the zone check.
``MPS_FIX*``, the macro/inline part of the fix operation, are referred
to as "fix stage 1" or "the first stage fix" in other documents and
comments.
If these inline checks pass, ``_mps_fix2`` is called. If the MPS has
been built as a separate object file or library, this is where the
function call out of the scan loop happens. Since version 1.110 of the
MPS, we encourage clients to compile the MPS in the same translation
unit as their format code, so that the compiler can be intelligent
about inlining parts of ``_mps_fix2`` in the format scanner. The
instructions for doing this are in `Building the Memory Pool System
<../manual/build.txt>`_, part of the manual.
The second stage fix in the MPM
-------------------------------
If a pointer gets past the first-stage fix filters, it is passed to
``_mps_fix2``, the "second stage fix". The second stage can filter out
yet more pointers using information about segments before it has to
consult the pool class.
The first test applied is the "tract test". The MPS looks up the tract
containing the address in the tract table, which is a simple linear
table indexed by the address shifted -- a kind of flat page table.
Note that if the arena has been extended, the tract table becomes less
simple, and this test may involved looking in more than one table.
This will cause a considerable slow-down in garbage collection
scanning. This is the reason that it's important to give a good
estimate of the amount of address space you will ever occupy with
objects when you initialize the arena.
The pointer might not even be in the arena (and so not in any tract).
The first stage fix doesn't guarantee it. So we eliminate any pointers
not in the arena at this stage.
If the pointer is in an allocated tract, then the table also contains
a cache of the "white set" -- the set of garbage collection traces for
which the tract is "interesting". If a tract isn't interesting, then
we know that it contains no condemned objects, and we can filter out
the pointer.
If the tract is interesting them it's part of a segment containing
objects that have been condemned. The MPM can't know anything about
the internal layout of the segment, so at this point we dispatch to
the third stage fix.
This dispatch is slightly subtle. We have a cache of the function to
dispatch to in the scan state, which has recently been looked at and
is with luck still in the processor cache. The reason there is a
dispatch at all is to allow for a fast changeover to emergency garbage
collection, or overriding of garbage collection with extra operations.
Those are beyond the scope of this document. Normally, ``ss->fix``
points at :c:func:`PoolFix()`, and we rely somewhat on modern processor
`branch target prediction
<https://en.wikipedia.org/wiki/Branch_target_predictor>`_).
:c:func:`PoolFix()` is passed the pool, which is fetched from the tract
table entry, and that should be in the cache.
:c:func:`PoolFix()` itself dispatches to the pool class. Normally, a
dispatch to a pool class would indirect through the pool class object.
That would be a double indirection from the tract, so instead we have
a cache of the pool's fix method in the pool object. This also allows
a pool class to vary its fix method per pool instance, a fact that is
exploited to optimize fixing in the AMC Pool depending on what kind of
object format it is managing.
The third stage fix in the pool class
-------------------------------------
The final stage of fixing is entirely dependent on the pool class. The
MPM can't, in general, know how the objects within a pool are arranged,
so this is pool class specific code.
Furthermore, the pool class must make decisions based on the "reference
rank" of the pointer. If a pointer is ambiguous (``RankAMBIG``) then it
can't be changed, so even a copying pool class can't move an object.
On the other hand, if the pointer is weak (``RankWEAK``) then the pool fix
method shouldn't preserve the object at all, even if it's condemned.
The exact details of the logic that the pool fix must implement in
order to co-operate with the MPM and other pools are beyond the scope
of this document, which is about the critical path. Since it is on
the critical path, it's important that whatever the pool fix does is
simple and fast and returns to scanning as soon as possible.
The first step, though, is to further filter out pointers which aren't
to objects, if that's its policy. Then, it may preserve the object,
according to its policy, and possibly ensure that the object gets
scanned at some point in the future, if it contains more pointers.
If the object is moved to preserve it (for instance, if the pool class
implements a copying GC), or was already moved when fixing a previous
reference to it, the reference being fixed must be updated (this is
the origin of the term "fix").
As a simple example, :c:func:`LOFix()` is the pool fix method for the Leaf
Only pool class. It implements a marking garbage collector, and does
not have to worry about scanning preserved objects because it is used
to store objects that don't contain pointers. (It is used in compiler
run-time systems to store binary data such as character strings, thus
avoiding any scanning, decoding, or remembered set overhead for them.)
:c:func:`LOFix()` filters any ambiguous pointers that aren't aligned, since
they can't point to objects it allocated. Otherwise it subtracts the
segment base address and shifts the result to get an index into a mark
bit table. If the object wasn't marked and the pointer is weak, then
it sets the pointer to zero, since the object is about to be recycled.
Otherwise, the mark bit is set, which preserves the object from
recycling when `LOReclaim` is called later on. `LOFix` illustrates
about the minimum and most efficient thing a pool fix method can do.
Other considerations
--------------------
So far this document has described the ways in which the garbage
collector is designed around optimising the critical path. There are a
few other things that the MPS does that are important.
Firstly, inlining is very important. The first stage fix is inlined
into the format scanner by being implemented in macros in
mps.h_. And to get even better inlining, `we
recommend <../manual/build.txt>`_ that the whole MPS is compiled in a
single translation unit with the client format and that strong global
optimisation is applied.
Secondly, we are very careful with code annotations on the critical
path. Assertions, statistics, and telemetry are all disabled on the
critical path in "hot" (production) builds. (In fact, it's because the
critical path is critical that we can afford to leave annotations
switched on elsewhere.)
Last, but by no means least, we pay a lot of brainpower and measurement
to the critical path, and are very very careful about changing it. Code
review around the critical path is especially vigilant.
And we write long documents about it.
References
----------
.. [MMRef] "The Memory Management Reference"; <http://www.memorymanagement.org/>.
.. _scanning: http://www.memorymanagement.org/glossary/s.html#scan
.. _marking: http://www.memorymanagement.org/glossary/m.html#marking
.. _copying: http://www.memorymanagement.org/glossary/c.html#copying.garbage.collection
.. _`"condemns"`: http://www.memorymanagement.org/glossary/c.html#condemned.set
.. _BIBOP: http://www.memorymanagement.org/glossary/b.html#bibop
.. _`remembered set`: http://www.memorymanagement.org/glossary/r.html#remembered.set
.. _`reference tag`: http://www.memorymanagement.org/glossary/t.html#tag
.. _`Open Dylan`: http://opendylan.org/

290
mps/manual/html/_sources/design/diag.txt Normal file
View file

@ -0,0 +1,290 @@
.. _design-diag:
.. index::
pair: diagnostic feedback; design
Diagnostic feedback
===================
.. mps:prefix:: design.mps.diag
Introduction
------------
:mps:tag:`intro` This document describes how to use the diagnostic feedback
mechanism in the Memory Pool System.
:mps:tag:`sources` Initially abased on [RHSK_2007-04-13]_ and [RHSK_2007-04-18]_.
Overview
--------
Diagnostic feedback is information created by the MPS diagnostic
system for the purpose of helping MPS programmers client-code
programmers.
Such a piece of information is called "a diagnostic". (See also
:mps:ref:`.parts`.)
A diagnostic is not intended to be end-user readable (or visible), or
machine-parseable.
A diagnostic is not intended to be stable from one release to the
next: it may be modified or removed at any time.
Requirements
------------
MPS diagnostic feedback code must do these things:
- calculate, store, and propagate data;
- collate, synthesise, and format it into a human-useful diagnostic;
- control (for example, filter) output of diagnostics;
- use a channel to get the diagnostic out.
Note: the knowledge/code/logic for constructing the human-useful
message is kept inside normal MPS source code. This means it is always
in-sync with changes to the MPS. This also means that any external
utilities used to display the messages do not need to understand, or
keep in sync with, the details of what's going inside the MPS.
Usage
-----
To run the MPS and get diagnostic output from it:
1. Use a variety with diagnostics compiled-in. Currently, that means
variety.di. See ``config.h``.
2. Check that the diagnostics you require are generated, by looking in
MPS source for invocations of the appropriate macro (for example,
:c:func:`DIAG_SINGLEF()`).
3. Check that the diagnostics you require will be output, by looking
at the diagnostic filter rules in ``diag.c``.
4. Run the MPS and client in an environment that supports the channel
used (for example, at a command-line if using :c:func:`WriteF()`).
What is a diagnostic?
.....................
A diagnostic has three parts:
1. a trigger condition, that causes this diagnostic to be emitted;
2. a text tag (for example, "TraceStart") which is the name of this
diagnostic; and
3. a paragraph of human-useful text.
A diagnostic is emitted by the MPS at a certain point in time when a
certain event happens.
Diagnostics are not nested. Every diagnostic must have a tag. Each
diagnostic should have a unique tag (uniqueness is just to help the
humans; the diagnostic system does not care).
The paragraph of text can be many lines long. It usually explains what
event caused the diagnostic to be emitted, and commonly also includes
the output of some :c:func:`Describe()` methods for various relevant
objects. (For example, the ``TraceStart`` diagnostic might call, and
include the output generated by, the :c:func:`TraceDescribe()` method).
How do I control (filter) which diagnostics I see?
..................................................
All diagnostics are emitted and then filtered according to the
"diagnostic filter rules".
The first level of control is filtering by tag. (For example, only
show ``TraceStart`` diagnostics).
The second level of control is filtering by paragraph content. (For
example, only show ``TraceStart`` diagnostics where the trace is
started because a nursery generation is full).
The third level of control is filtering by line content. (For example,
only show lines containing the word ``whiteSet``).
See ``diag.c`` for details.
Note: the entire filtering mechanism can be turned off, so that
diagnostics go immediately to ``mps_lib_get_stdout(0``, with no
buffering or filtering See impl.c.diag.filter-disable.
How to write a diagnostic
-------------------------
Improve stateless Describe methods where possible
.................................................
Where possible, don't put clever code into an event-triggered
diagnostic: put it into a stateless :c:func:`Describe()` method instead, and
then call that method when emitting your diagnostic.
For example::
FooDescribe(Foo foo, mps_lib_FILE *stream)
{
/* show value of new "quux" field */
WriteF(stream, "Foo: $P { quux: $U }\n", foo, foo->quux);
}
FooWibble(Foo foo)
{
...
DIAG_FIRSTF(( "FooWibble", "Wibbling foo $P", foo, NULL));
DIAG( FooDescribe(foo, DIAG_STREAM); );
DIAG_END("FooWibble");
...
}
This is much better, because other people can use your human-useful
output in their diagnostics, or 'live' in a debugger.
Use the output macros
.....................
For a simple diagnostic, use :c:func:`DIAG_SINGLEF()`. This begins the tag,
puts text into the paragraph, and ends the tag immediately.
For a more complex diagnostic, the first call must be
:c:func:`DIAG_FIRSTF()`, which begins a diag tag.
While a tag is current, you can add text to the diagnostic's paragraph
using :c:func:`DIAG_MOREF()`, and ``WriteF( DIAG_STREAM, ... )``.
.. note::
:c:macro:`DIAG_STREAM` is not a real standard C library stream. If you
want stream-level access, you may use :c:func:`Stream_fputc()` and
:c:func:`Stream_fputs()`.
End the tag by calling :c:macro:`DIAG_END`.
Compile away in non-diag varieties; no side effects
...................................................
Wrap non-output code with the :c:func:`DIAG()` and :c:func:`DIAG_DECL()` macros,
to make sure that non-diag varieties do not execute
diagnostic-generating code.
For complex diagnostic-generating code, it may be cleaner to move it
into a separate local function. Put ``_diag`` on the end of the function
name (for example, :c:func:`TraceStart_diag()`).
Obviously, diagnostic-generating code must have no side effects.
Choosing tags
.............
Tags should be valid C identifiers. Unless you know of a good reason
why not. (Not currently checked).
There's no formal scheme for tag naming, but make it helpful and
informally hierarchical, for example, ``TraceBegin``, ``TraceStart``,
``TraceEnd``, and so on, not ``BeginTrace``, ``EndTrace``.
Writing good paragraph text
...........................
IMPORTANT: Make your diagnostics easy to understand! Other people will
read your diagnostics! Make them clear and helpful. Do not make them
terse and cryptic. If you use symbols, print a key in the diagnostic.
(If you don't want to see this the screen clutter, then you can always
add a filter rule to your personal rule set to filter it out).
Maintaining helpful filter rules
................................
If you add a noisy diagnostic, add a rule to the default ruleset to
turn it off.
How the MPS diagnostic system works
-----------------------------------
Channels
........
The recommended channel is :c:func:`WriteF()` to standard output.
Other possible of future channels might be:
- :c:func:`printf()`;
- a new type (yet to be defined) of ``mps_message``;
- squirt them into the telemetry-log-events system;
- telnet.
Currently, only :c:func:`printf()` and :c:func:`WriteF()` are supported. See the
:c:macro:`DIAG_WITH_` macros in ``mpm.h``.
You can also use a debugger to call :c:func:`Describe()` methods directly,
from within the debugger.
Note: it is unfortunate that choice of channel may (for some channels)
also dictate the form of the code that synthesises the message. (For
example, :c:func:`WriteF()` style parameter-expansion is not possible when
using the :c:func:`printf()` channel, because there is no way to get
:c:func:`WriteF()` to produce its output into a string). This is just a
technical hitch; logically, the code that synthesises a diagnostic
message should not care which channel will be used to transmit it out
of the MPS.
Parts of the MPS diagnostic system
..................................
:mps:tag:`parts` The following facilities are considered part of the MPS
diagnostic system:
- the :c:func:`Describe()` methods.
- the :c:macro:`DIAG` macros (:c:macro:`DIAG`, :c:macro:`DIAG_DECL`, ``DIAG_*F``, and so on);
- the :c:macro:`STATISTIC` macros (see ``mpm.h``);
- the :c:macro:`METER` macros and meter subsystem.
Related systems
...............
The MPS diagnostic system is separate from the following other MPS
systems:
- The telemetry-log-events system. This emits much more data, in a
less human-readable form, requires MPS-aware external tools, and is
more stable from release to release). In non-diagnostic telemetry
varieties, the telemetry-log-events system emits events that log all
normal MPS actions. In diagnostic telemetry varieties, it may emit
additional events containing diagnostic information. Additionally,
the telemetry-log-events stream might in future be available as a
channel for emitting human-readable text diagnostics. See also
design.mps.telemetry.
- The MPS message system. This is present in all varieties, and
manages asynchronous communication from the MPS to the client
program). However, the MPS message system might in future also be
available as a channel for emitting diagnostics. See also
design.mps.message.
References
----------
.. [RHSK_2007-04-13] Richard Kistruck. 2007-04-13. "`diagnostic feedback from the MPS <https://info.ravenbrook.com/mail/2007/04/13/13-07-45/0.txt>`_".
.. [RHSK_2007-04-18] Richard Kistruck. 2007-04-18. "`Diverse types of diagnostic feedback <http://info.ravenbrook.com/mail/2007/04/18/10-58-49/0.txt>`_".

118
mps/manual/html/_sources/design/finalize.txt
View file

@ -1,6 +1,120 @@
.. _design-finalize:
.. index::
pair: finalization; design
.. _design-finalize:
.. include:: ../../converted/finalize.rst
Finalization
============
.. mps:prefix:: design.mps.finalize
Overview
--------
:mps:tag:`overview` Finalization is implemented internally using the
Guardian Pool Class (design.mps.poolmrg). Objects can be registered
for finalization using an interface function (called
:c:func:`mps_finalize()`). Notification of finalization is given to the client
via the messaging interface. ``PoolClassMRG`` (design.mps.poolmrg)
implements a Message Class which implements the finalization messages.
Requirements
------------
:mps:tag:`req` Historically only Dylan had requirements for finalization,
see req.dylan.fun.final. Now (2003-02-19) Configura have requirements
for finalization. Happily they are very similar.
Architecture
------------
External interface
..................
:mps:tag:`if.register` :c:func:`mps_finalize()` increases the number of times that
an object has been registered for finalization by one. The object must
have been allocated from the arena (space). Any finalization messages
that are created for this object will appear on the arena's message
queue. The MPS will attempt to finalize the object that number of
times.
:mps:tag:`if.deregister` :c:func:`mps_definalize()` reduces the number of times that
the object located at ``obj`` has been registered for finalization by
one. It is an error to definalize an object that has not been
registered for finalization.
:mps:tag:`if.deregister.not` At the moment (1997-08-20) :c:func:`mps_definalize()`
is not implemented.
:mps:tag:`if.get-ref` :c:func:`mps_message_finalization_ref()` returns the reference
to the finalized object stored in the finalization message.
Implementation
--------------
:mps:tag:`int.over` Registering an object for finalization corresponds to
allocating a reference of rank FINAL to that object. This reference is
allocated in a guardian object in a pool of ``PoolClassMRG`` (see
design.mps.poolmrg).
:mps:tag:`int.arena.struct` The MRG pool used for managing final references
is kept in the Arena (Space), referred to as the "final pool".
:mps:tag:`int.arena.lazy` The pool is lazily created. It will not be created
until the first object is registered for finalization.
:mps:tag:`int.arena.flag` There is a flag in the Arena that indicates
whether the final pool has been created yet or not.
.. c:function:: Res ArenaFinalize(Arena arena, Ref addr)
:mps:tag:`int.finalize.create` Creates the final pool if it has not been
created yet.
:mps:tag:`int.finalize.alloc` Allocates a guardian in the final pool.
:mps:tag:`int.finalize.write` Writes a reference to the object into the
guardian object.
:mps:tag:`int.finalize.all` That's all.
:mps:tag:`int.finalize.error` If either the creation of the pool or the
allocation of the object fails then the error will be reported back to
the caller.
:mps:tag:`int.finalize.error.no-unwind` This function does not need to do
any unwinding in the error cases because the creation of the pool is
not something that needs to be undone.
:mps:tag:`int.arena-destroy.empty` :c:func:`ArenaDestroy()` empties the message
queue by calling :c:func:`MessageEmpty()`.
:mps:tag:`int.arena-destroy.final-pool` If the final pool has been created
then :c:func:`ArenaDestroy()` destroys the final pool.
:mps:tag:`access` :c:func:`mps_message_finalization_ref()` needs to access the
finalization message to retrieve the reference and then write it to
where the client asks. This must be done carefully, in order to avoid
breaking the invariants or creating a hidden root.
:mps:tag:`access.invariants` We protect the invariants by using special
routines :c:func:`ArenaRead()` and :c:func:`ArenaPoke()` to read and write the
reference. This works as long as there's no write-barrier collection.
.. note::
Instead of :c:func:`ArenaPoke()`, we could put in an :c:func:`ArenaWrite()`
that would be identical to :c:func:`ArenaPoke()`, except that it would
:c:func:`AVER()` the invariant (or it can just :c:func:`AVER()` that there are
no busy traces unflipped). When we get write-barrier collection,
we could change it to do the real thing, but in the absence of a
write-barrier, it's functionally identical to :c:func:`ArenaPoke()`.
Pekka P. Pirinen, 1997-12-09.

54
mps/manual/html/_sources/design/fix.txt
View file

@ -1,6 +1,56 @@
.. _design-fix:
.. index::
pair: fix function; design
.. _design-fix:
.. include:: ../../converted/fix.rst
The generic fix function
========================
.. mps:prefix:: design.mps.fix
Introduction
-------------
:mps:tag:`intro` Fix is the interface through which the existence of
references are communicated from the MPS client to the MPS. The
interface also allows the value of such references to be changed (this
is necessary in order to implement a moving memory manager).
Architecture
-------------
:mps:tag:`protocol.was-marked` The :c:type:`ScanState` has a :c:type:`Bool`
``wasMarked`` field. This is used for finalization.
:mps:tag:`protocol.was-marked.set` All pool-specific fix methods must set
the ``wasMarked`` field in the :c:type:`ScanState` that they are passed.
:mps:tag:`protocol.was-marked.meaning` If the pool-specific fix method sets
the ``wasMarked`` field to :c:macro:`FALSE` it is indicating the object
referred to by the ref (the one that it is supposed to be fixing) has
not previously been marked (ie, this is the first reference to this
object that has been fixed), and that the object was white (in
condemned space).
:mps:tag:`protocol.was-marked.conservative` It is always okay to set the
``wasMarked`` field to :c:macro:`TRUE`.
:mps:tag:`protocol.was-marked.finalizable` The MRG pool (design.mps.poolmrg)
uses the value of the ``wasMarked`` field to determine whether an
object is finalizable.
Implementation
---------------
:mps:tag:`fix.nailed` In a copying collection, a non-ambiguous fix to a
broken heart should be snapped out *even if* there is a ``RankAMBIG``
ref to same object (that is, if the broken heart is nailed); the
``RankAMBIG`` reference must either be stale (no longer in existence)
or bogus.

100
mps/manual/html/_sources/design/guide.hex.trans.txt
View file

@ -1,6 +1,102 @@
.. _design-guide.hex.trans:
.. index::
pair: hexadecimal; transliterating
.. _design-guide.hex.trans:
.. include:: ../../converted/guide.hex.trans.rst
Transliterating the alphabet into hexadecimal
=============================================
.. mps:prefix:: guide.hex.trans
Introduction
------------
:mps:tag:`scope` This document explains how to represent the alphabet as
hexadecimal digits.
:mps:tag:`readership` This document is intended for anyone devising
arbitrary constants which may appear in hex-dumps.
:mps:tag:`sources` This transliteration was supplied by RichardK in
mail.richardk.1997-04-07.13-44.
Transliteration
---------------
:mps:tag:`forward` The chosen transliteration is as follows::
ABCDEFGHIJKLMNOPQRSTUVWXYZ
ABCDEF9811C7340BC6520F3812
:mps:tag:`backward` The backwards transliteration is as follows::
0 OU
1 IJY
2 TZ
3 MW
4 N
5 S
6 R
7 L
8 HX
9 G
A A
B BP
C CKQ
D D
E E
F FV
:mps:tag:`pad` If padding is required (to fill a hex constant length), you
should use 9's, because G is rare and can usually be inferred from
context.
:mps:tag:`punc` There is no formal scheme for spaces, or punctuation. It is
suggested that you use 9 (as :mps:ref:`.pad`).
Justification
--------------
:mps:tag:`letters` The hexadecimal letters (A-F) are all formed by
similarity of sound. B and P sound similar, as do F and V, and C, K, &
Q can all sound similar.
:mps:tag:`numbers` The numbers (0-9) are all formed by similarity of shape
(but see :mps:ref:`.trans.t`). Nevertheless, 1=IJY retains some similarity of
sound.
:mps:tag:`trans.t` T is an exception to :mps:ref:`.numbers`, but is such a common
letter that it deserves it.
Notes
-----
:mps:tag:`change` This transliteration differs from the old transliteration
used for signatures (see design.mps.sig(0)), as follows: J:6->1;
L:1->7; N:9->4; R:4->6; W:8->3; X:5->8; Y:E->I.
:mps:tag:`problem.mw` There is a known problem that M and W are both common,
map to the same digit (3), and are hard to distinguish in context.
:mps:tag:`find.c` It is possible to find all 8-digit hexadecimal constants
and how many times they're used in C files, using the following Perl
script::
perl5 -n -e 'BEGIN { %C=(); } if(/0x([0-9A-Fa-f]{8})/) { $C{$1} = +[] if(
!defined($C{$1})); push(@{$C{$1}}, $ARGV); } END { foreach $H (sort(keys(%C)))
{ printf "%3d %s %s\n", scalar(@{$C{$H}}), $H, join(", ", @{@C{$H}}); } }' *.c
*.h
:mps:tag:`comment` It is a good idea to add a comment to any constant
declaration indicating the English version and which letters were
selected (by capitalisation), e.g.::
#define SpaceSig ((Sig)0x5195BACE) /* SIGnature SPACE */

427
mps/manual/html/_sources/design/guide.impl.c.format.txt Normal file
View file

@ -0,0 +1,427 @@
.. _design-guide.impl.c.format:
.. index::
pair: C language; formatting guide
pair: C language formatting; guide
C Style -- formatting
=====================
.. mps:prefix:: guide.impl.c.format
pair: C language; formatting guide
pair: C language formatting; guide
Introduction
------------
:mps:tag:`scope` This document describes the Ravenbrook conventions for the
general format of C source code in the MPS.
:mps:tag:`readership` This document is intended for anyone working on or with the
C source code.
General Formatting Conventions
------------------------------
Line Width
..........
:mps:tag:`width` Lines should be no wider than 72 characters. :mps:tag:`width.why` Many
people use 80 column terminal windows so that multiple windows can be
placed side by side. Restricting lines to 72 characters allows line
numbering to be used (in vi for example) and also allows diffs to be
displayed without overflowing the terminal.
White Space
...........
:mps:tag:`space.notab` No tab characters should appear in the source files.
Ordinary spaces should be used to indent and format the sources.
:mps:tag:`space.notab.why` Tab characters are displayed differently on different
platforms, and sometimes translated back and forth, destroying layout
information.
:mps:tag:`space.punct` There should always be whitespace after commas and
semicolons and similar punctuation.
:mps:tag:`space.op` Put white space around operators in expressions, except when
removing it would make the expression clearer by binding certain
sub-expressions more tightly. For example::
foo = x + y*z;
:mps:tag:`space.control` One space between the keyword, ``switch``, ``while``,
``for`` and the following paren. :mps:tag:`space.control.why` This distinguishes
control statements lexically from function calls, making it easier to
distinguish them visually and when searching with tools like grep.
:mps:tag:`space.function.not` No space between a function name and the following
paren beginning its argument list.
Sections and Paragraphs
.......................
:mps:tag:`section` Source files can be thought of as breaking down into
"sections" and "paragraphs". A section might be the leader comment of a
file, the imports, or a set of declarations which are related.
:mps:tag:`section.space` Precede sections by two blank lines (except the first
one in the file, which should be the leader comment in any case).
:mps:tag:`section.comment` Each section should start with a banner comment (see
.comment.banner) describing what the section contains.
:mps:tag:`para` Within sections, code often breaks down into natural units called
"paragraphs". A paragraph might be a set of strongly related
declarations (Init and Finish, for example), or a few lines of code
which it makes sense to consider together (the assignment of fields into
a structure, for example).
:mps:tag:`para.space` Precede paragraphs by a single blank line.
Statements
..........
:mps:tag:`statement.one` Generally only have at most one statement per line. In
particular the following are deprecated::
if (thing) return;
a=0; b=0;
case 0: f = inRampMode ? AMCGen0RampmodeFrequency : AMCGen0Frequency;
:mps:tag:`statement.one.why` Debuggers can often only place breakpoints on lines,
not expressions or statements within a line. The ``if (thing) return;`` is
a particularly important case, if thing is a reasonably rare return
condition then you might want to breakpoint it in a debugger session.
Annoying because ``if (thing) return;`` is quite compact and pleasing
otherwise.
Indentation
...........
:mps:tag:`indent` Indent the body of a block by two spaces. For formatting
purposes, the "body of a block" means:
- statements between braces,
- a single statement following a lone ``if``;
- statements in a switch body; see .switch.
(:mps:tag:`indent.logical` The aim is to group what we think of as logical
blocks, even though they may not exactly match how "block" is used in
the definition of C syntax).
Some examples::
if (res != ResOK) {
SegFinish(&span->segStruct);
PoolFreeP(MV->spanPool, span, sizeof(SpanStruct));
return res;
}
if (res != ResOK)
goto error;
if (j == block->base) {
if (j+step == block->limit) {
if (block->thing)
putc('@', stream);
}
} else if (j+step == block->limit) {
putc(']', stream);
pop_bracket();
} else {
putc('.', stream);
}
switch (c) {
case 'A':
c = 'A';
p += 1;
break;
}
:mps:tag:`indent.goto-label` Place each goto-label on a line of its own,
outdented to the same level as the surrounding block. Then indent the
non-label part of the statement normally.
::
result foo(void)
{
statement();
if (error)
goto foo;
statement();
return OK;
foo:
unwind();
return ERROR;
}
:mps:tag:`indent.case-label` Outdent case- and default-labels in a switch
statement in the same way as :mps:ref:`.indent.goto-label`. See :mps:ref:`.switch`.
:mps:tag:`indent.cont` If an expression or statement won't fit on a single line,
indent the continuation lines by two spaces, apart from the following
exception:
:mps:tag:`indent.cont.parens` if you break a statement inside a parameter list or
other parenthesized expression, indent so that the continuation lines up
just after the open parenthesis. For example::
PoolClassInit(&PoolClassMVStruct,
"MV", init, finish, allocP, freeP,
NULL, NULL, describe, isValid);
:mps:tag:`indent.cont.expr` Note that when breaking an expression it is clearer
to place the operator at the start of the continuation line::
CHECKL(AddrAdd((Addr)chunk->pageTableMapped,
BTSize(chunk->pageTablePages))
<= AddrAdd(chunk->base, chunk->ullageSize));
This is particularly useful in long conditional expressions that use &&
and ||. For example::
} while(trace->state != TraceFINISHED
&& (trace->emergency || traceWorkClock(trace) < pollEnd));
:mps:tag:`indent.hint` Usually, it is possible to determine the correct
indentation for a line by looking to see if the previous line ends with
a semicolon. If it does, indent to the same amount, otherwise indent by
two more spaces. The main exceptions are lines starting with a close
brace, goto-labels, and line-breaks between parentheses.
Positioning of Braces
.....................
:mps:tag:`brace.otb` Use the "One True Brace" (or OTB) style. This places the
open brace after the control word or expression, separated by a space,
and when there is an else, places that after the close brace. For
example::
if(isBase) {
new->base = limit;
new->limit = block->limit;
block->limit = base;
new->next = block->next;
block->next = new;
} else {
new->base = block->base;
new->limit = base;
block->base = limit;
new->next = block;
*prev = new;
}
The same applies to struct, enum, union.
:mps:tag:`brace.otb.function.not` OTB is never used for function definitions.
:mps:tag:`brace.always` Braces are always required after ``if``, ``else``, ``switch``,
``while``, ``do``, and ``for``.
:mps:tag:`brace.always.except` Except that a lone ``if`` with no ``else`` is allowed
to drop its braces when its body is a single simple statement. Typically
this will be a ``goto`` or an assignment. For example::
if (res != ResOK)
goto failStart;
Note in particular that an ``if`` with an ``else`` must have braces on both
paths.
Switch Statements
.................
:mps:tag:`switch` format switch statements like this::
switch (action) {
case WIBBLE:
case WOBBLE:
{
int angle;
err = move(plate, action, &angle);
}
break;
case QUIET:
drop();
/* fall-through */
case QUIESCENT:
err = 0;
break;
default:
NOTREACHED;
break;
}
The component rules that result in this style are:
:mps:tag:`switch.break` The last line of every case-clause body must be an
unconditional jump statement (usually ``break``, but may be ``goto``,
``continue``, or ``return``), or if a fall-through is intended, the comment
``/* fall-through */``. (Note: if the unconditional jump should never be
taken, because of previous conditional jumps, use :c:macro:`NOTREACHED` on the
line before it). This rule is to prevent accidental fall-throughs, even
if someone makes a editing mistake that causes a conditional jump to be
missed.
:mps:tag:`switch.default` It is usually a good idea to have a default-clause,
even if all it contains is :c:macro:`NOTREACHED` and ``break``. Remember that
:c:macro:`NOTREACHED` doesn't stop the process in all build varieties.
Formatting Comments
...................
:mps:tag:`comment` There are three types of comments: banners, paragraph
comments, and columnar comments.
:mps:tag:`comment.banner` Banner comments come at the start of sections. A banner
comment consists of a heading usually composed of a symbol, an em-dash
(--) and an short explanation, followed by English text which is
formatted using conventional text documentation guidelines (see
guide.text). The open and close comment tokens (``/*`` and ``*/``) are
placed at the top and bottom of a column of asterisks. The text is
separated from the asterisks by one space. Place a blank line between
the banner comment and the section it comments. For example::
/* BlockStruct -- Block descriptor
*
* The pool maintains a descriptor structure for each
* contiguous allocated block of memory it manages.
* The descriptor is on a simple linked-list of such
* descriptors, which is in ascending order of address.
*/
typedef struct BlockStruct {
:mps:tag:`comment.para` Paragraph comments come at the start of paragraphs in the
code. A paragraph comment consists of formatted English text, with each
line wrapped by the open and close comment tokens (``/*`` and ``*/``).
(This avoids problems when cutting and pasting comments.) For example::
/* If the freed area is in the base sentinel then insert */
/* the new descriptor after it, otherwise insert before. */
if(isBase) {
:mps:tag:`comment.para.precede` Paragraph comments, even one-liners, precede the
code to which they apply.
:mps:tag:`comment.column` Columnar comments appear in a column to the right of
the code. They should be used sparingly, since they clutter the code and
make it hard to edit. Use them on variable declarations and structure,
union, or enum declarations. They should start at least at column 32
(counting from 0, that is, on a tab-stop), and should be terse
descriptive text. Abandon English sentence structure if this makes the
comment clearer. Don't write more than one line. Here's an example::
typedef struct PoolMVStruct {
Pool blockPool; /* for block descriptors */
Pool spanPool; /* for span descriptors */
size_t extendBy; /* size to extend pool by */
size_t avgSize; /* estimate of allocation size */
size_t maxSize; /* estimate of maximum size */
Addr space; /* total free space in pool */
Addr lost; /* lost when free can't allocate */
struct SpanStruct *spans; /* span chain */
} PoolMVStruct;
Macros
......
:mps:tag:`macro.careful` Macros in C are a real horror bag, be extra careful.
There's lots that could go here, but proper coverage probably deserves a
separate document. Which isn't written yet.
:mps:tag:`macro.general` Do try and follow the other formatting conventions for
code in macro definitions.
:mps:tag:`macro.backslash` Backslashes used for continuation lines in macro
definitions should be put on the right somewhere where they will be less
in the way. Example::
#define RAMP_RELATION(X) \
X(RampOUTSIDE, "outside ramp") \
X(RampBEGIN, "begin ramp") \
X(RampRAMPING, "ramping") \
X(RampFINISH, "finish ramp") \
X(RampCOLLECTING, "collecting ramp")
History
-------
- 2007-06-04 DRJ_ Adopted from Harlequin MMinfo version and edited.
- 2007-06-04 DRJ_ Changed .width from 80 to 72. Banned space between
``if`` and ``(``. Required braces on almost everything. Clarified that
paragraph comments precede the code.
- 2007-06-13 RHSK_ Removed .brace.block, because MPS source always
uses .brace.otb. Remove .indent.elseif because it is obvious (ahem) and
showing an example is sufficient. New rules for .switch.*: current MPS
practice is a mess, so lay down a neat new law.
- 2007-06-27 RHSK_ Added :mps:ref:`.space.function.not`.
- 2007-07-17 DRJ_ Added .macro.\*
- 2012-09-26 RB_ Converted to Markdown and reversed inconsistent
switch "law".
.. _DRJ: http://www.ravenbrook.com/consultants/drj
.. _RHSK: http://www.ravenbrook.com/consultants/rhsk
.. _RB: http://www.ravenbrook.com/consultants/rb
Copyright and License
---------------------
This document is copyright © 2002-2012 [Ravenbrook
Limited](http://www.ravenbrook.com/). All rights reserved. This is an
open source license. Contact Ravenbrook for commercial licensing
options.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Redistributions in any form must be accompanied by information on
how to obtain complete source code for this software and any
accompanying software that uses this software. The source code must
either be included in the distribution or be available for no more
than the cost of distribution plus a nominal fee, and must be freely
redistributable under reasonable conditions. For an executable file,
complete source code means the source code for all modules it
contains. It does not include source code for modules or files that
typically accompany the major components of the operating system on
which the executable file runs.
**This software is provided by the copyright holders and contributors
"as is" and any express or implied warranties, including, but not
limited to, the implied warranties of merchantability, fitness for a
particular purpose, or non-infringement, are disclaimed. In no event
shall the copyright holders and contributors be liable for any direct,
indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or
services; loss of use, data, or profits; or business interruption)
however caused and on any theory of liability, whether in contract,
strict liability, or tort (including negligence or otherwise) arising in
any way out of the use of this software, even if advised of the
possibility of such damage.**

1
mps/manual/html/_sources/design/index.txt
View file

@ -9,6 +9,7 @@ Design
config
critical-path
guide.hex.trans
guide.impl.c.format
keyword-arguments
ring
sig

331
mps/manual/html/_sources/design/interface-c.txt Normal file
View file

@ -0,0 +1,331 @@
.. _design-interface-c:
.. index::
pair: C interface; design
C interface design
==================
.. mps:prefix:: design.mps.interface.c
Introduction
------------
:mps:tag:`scope` This document is the design for the Memory Pool System
(MPS) interface to the C Language, impl.h.mps.
:mps:tag:`bg` See mail.richard.1996-07-24.10-57.
Analysis
--------
Goals
.....
:mps:tag:`goal.c` The file impl.h.mps is the C external interface to the
MPS. It is the default interface between client code written in C and
the MPS. :mps:tag:`goal.cpp` impl.h.mps is not specifically designed to be
an interface to C++, but should be usable from C++.
Requirements
............
:mps:tag:`req` The interface must provide an interface from client code
written in C to the functionality of the MPS required by the product
(see req.product), Dylan (req.dylan), and the Core RIP (req.epcore).
``mps.h`` may not include internal MPS header files (such as
``pool.h``).
It is essential that the interface cope well with change, in order to
avoid restricting possible future MPS developments. This means that
the interface must be "open ended" in its definitions. This accounts
for some of the apparently tortuous methods of doing things
(``mps_fmt_A_t``, for example). The requirement is that the MPS should
be able to add new functionality, or alter the implementation of
existing functionality, without affecting existing client code. A
stronger requirement is that the MPS should be able to change without
*recompiling* client code. This is not always possible.
.. note::
`.naming.global` was presumably done in response to unwritten
requirements regarding the use of the name spaces in C, perhaps
these:
- :mps:tag:`req.name.iso` The interface shall not conflict in terms of
naming with any interfaces specified by ISO C and all reasonable
future versions.
- :mps:tag:`req.name.general` The interface shall use a documented and
reasonably small portion of the namespace so that clients can
interoperate easily.
David Jones, 1998-10-01.
Architecture
------------
:mps:tag:`fig.arch` The architecture of the MPS Interface
[missing figure]
Just behind ``mps.h`` is the file ``mpsi.c``, the "MPS interface
layer" which does the job of converting types and checking parameters
before calling through to the MPS proper, using internal MPS methods.
General conventions
-------------------
:mps:tag:`naming` The external interface names should adhere to the
documented interface conventions; these are found in
doc.mps.ref-man.if-conv(0).naming. They are paraphrased/recreated
here.
:mps:tag:`naming.unixy` The external interface does not follow the same
naming conventions as the internal code. The interface is designed to
resemble a more conventional C, Unix, or Posix naming convention.
:mps:tag:`naming.case` Identifiers are in lower case, except
non-function-like macros, which are in upper case.
:mps:tag:`naming.global` All publicised identifiers are
prefixed ``mps_`` or :c:macro:`MPS_`.
:mps:tag:`naming.all` All identifiers defined by the MPS
should begin ``mps_`` or :c:macro:`MPS_` or ``_mps_``.
:mps:tag:`naming.type` Types are suffixed ``_t``.
:mps:tag:`naming.struct` Structure types and tags are suffixed ``_s``.
:mps:tag:`naming.union` Unions types and tags are suffixed ``_u``.
:mps:tag:`naming.scope` The naming conventions apply to all identifiers (see
ISO C §6.1.2); this includes names of functions, variables, types
(through typedef), structure and union tags, enumeration members,
structure and union members, macros, macro parameters, labels.
:mps:tag:`naming.scope.labels` labels (for ``goto`` statements) should be
rare, only in special block macros and probably not even then.
.. note::
This principle is not adhered to in the source code, which uses
``goto`` for handling error cases. Gareth Rees, 2013-05-27.
:mps:tag:`naming.scope.other` The naming convention would also extend to
enumeration types and parameters in functions prototypes but both of
those are prohibited from having names in an interface file.
:mps:tag:`type.gen` The interface defines memory addresses as ``void *`` and
sizes as ``size_t`` for compatibility with standard C (in particular,
with :c:func:`malloc()`). These types must be binary compatible with the
internal types :c:type:`Addr` and :c:type:`Size` respectively. Note that this
restricts the definitions of the internal types :c:type:`Addr` and :c:type:`Size`
when the MPS is interfaced with C, but does not restrict the MPS in
general.
:mps:tag:`type.opaque` Opaque types are defined as pointers to structures
which are never defined. These types are cast to the corresponding
internal types in ``mpsi.c``.
:mps:tag:`type.trans` Some transparent structures are defined. The client is
expected to read these, or poke about in them, under restrictions
which should be documented. The most important is probably the
allocation point (:c:type:`mps_ap_s`) which is part of allocation buffers.
The transparent structures must be binary compatible with
corresponding internal structures. For example, the fields of
:c:type:`mps_ap_s` must correspond with :c:type:`APStruct` internally. This is
checked by ``mpsi.c`` in :c:func:`mps_check()`.
:mps:tag:`type.pseudo` Some pseudo-opaque structures are defined. These only
exist so that code can be inlined using macros. The client code
shouldn't mess with them. The most important case of this is the scan
state (:c:type:`mps_ss_s`) which is accessed by the in-line scanning macros,
``MPS_SCAN_*`` and ``MPS_FIX*``.
:mps:tag:`type.enum` There should be no enumeration types in the interface.
Note that enum specifiers (to declare integer constants) are fine as
long as no type is declared. See guide.impl.c.misc.enum.type.
:mps:tag:`type.fun` Whenever function types or derived function types (such
as pointer to function) are declared a prototype should be used and
the parameters to the function should not be named. This includes the
case where you are declaring the prototype for an interface function.
:mps:tag:`type.fun.example` So use::
extern mps_res_t mps_alloc(mps_addr_t *, mps_pool_t, size_t, ...);
rather than::
extern mps_res_t mps_alloc(mps_addr_t *addr_return, mps_pool_t pool , size_t size, ...);
and::
typedef mps_addr_t (*mps_fmt_class_t)(mps_addr_t);
rather than::
typedef mps_addr_t (*mps_fmt_class_t)(mps_addr_t object);
See guide.impl.c.misc.prototype.parameters.
Checking
--------
:mps:tag:`check.space` When the arena needs to be recovered from a parameter
it is check using ``AVERT(Foo, foo)`` before any attempt to call
``FooArena(foo)``. The macro :c:func:`AVERT()` in impl.h.assert performs
simple thread-safe checking of ``foo``, so it can be called outside of
:c:func:`ArenaEnter()` and :c:func:`ArenaLeave()`.
:mps:tag:`check.types` We use definitions of types in both our external
interface and our internal code, and we want to make sure that they
are compatible. (The external interface changes less often and hides
more information.) At first, we were just checking their sizes, which
wasn't very good, but I've come up with some macros which check the
assignment compatibility of the types too. This is a sufficiently
useful trick that I thought I'd send it round. It may be useful in
other places where types and structures need to be checked for
compatibility at compile time.
These macros don't generate warnings on the compilers I've tried.
.. c:function:: COMPATLVALUE(lvalue1, lvalue2)
This macro checks the assignment compatibility of two lvalues. It uses
:c:func:`sizeof()` to ensure that the assignments have no effect. ::
#define COMPATLVALUE(lv1, lv2) \
((void)sizeof((lv1) = (lv2)), (void)sizeof((lv2) = (lv1)), TRUE)
.. c:function:: COMPATTYPE(type1, type2)
This macro checks that two types are assignment-compatible and equal
in size. The hack here is that it generates an lvalue for each type by
casting zero to a pointer to the type. The use of :c:func:`sizeof()` avoids
the undefined behaviour that would otherwise result from dereferencing
a null pointer. ::
#define COMPATTYPE(t1, t2) \
(sizeof(t1) == sizeof(t2) && \
COMPATLVALUE(*((t1 *)0), *((t2 *)0)))
.. c:function:: COMPATFIELDAPPROX(structure1, field1, structure2, field2)
This macro checks that the offset and size of two fields in two
structure types are the same. ::
#define COMPATFIELDAPPROX(s1, f1, s2, f2) \
(sizeof(((s1 *)0)->f1) == sizeof(((s2 *)0)->f2) && \
offsetof(s1, f1) == offsetof(s2, f2))
.. c:function:: COMPATFIELD(structure1, field1, structure2, field2)
This macro checks the offset, size, and assignment-compatibility of
two fields in two structure types. ::
#define COMPATFIELD(s1, f1, s2, f2) \
(COMPATFIELDAPPROX(s1, f1, s2, f2) && \
COMPATLVALUE(((s1 *)0)->f1, ((s2 *)0)->f2))
Binary compatibility issues
---------------------------
As in, "Enumeration types are not allowed" (see
mail.richard.1995-09-08.09-28).
There are two main aspects to run-time compatibility: binary interface
and protocol. The binary interface is all the information needed to
correctly use the library, and includes external symbol linkage,
calling conventions, type representation compatibility, structure
layouts, etc. The protocol is how the library is actually used by the
client code -- whether this is called before that -- and determines
the semantic correctness of the client with respect to the library.
The binary interface is determined completely by the header file and
the target. The header file specifies the external names and the
types, and the target platform specifies calling conventions and type
representation. There is therefore a many-to-one mapping between the
header file version and the binary interface.
The protocol is determined by the implementation of the library.
Constraints
-----------
:mps:tag:`cons` The MPS C Interface constrains the MPS in order to provide
useful memory management services to a C or C++ program.
:mps:tag:`cons.addr` The interface constrains the MPS address type, Addr
(design.mps.type.addr), to being the same as C's generic pointer type,
``void *``, so that the MPS can manage C objects in the natural way.
:mps:tag:`pun.addr` We pun the type of :c:type:`mps_addr_t` (which is ``void *``)
into :c:type:`Addr` (an incomplete type, see design.mps.type.addr). This
happens in the call to the scan state's fix function, for example.
:mps:tag:`cons.size` The interface constrains the MPS size type, :c:type:`Size`
(design.mps.type.size), to being the same as C's size type,
``size_t``, so that the MPS can manage C objects in the natural way.
:mps:tag:`pun.size` We pun the type of ``size_t`` in mps.h into :c:type:`Size` in
the MPM, as an argument to the format methods. We assume this works.
:mps:tag:`cons.word` The MPS assumes that :c:type:`Word` (design.mps.type.word)
and :c:type:`Addr` (design.mps.type.addr) are the same size, and the
interface constrains :c:type:`Word` to being the same size as C's generic
pointer type, ``void *``.
Notes
-----
The file ``mpstd.h`` is the MPS target detection header. It decodes
preprocessor symbols which are predefined by build environments in
order to determine the target platform, and then defines uniform
symbols, such as :c:macro:`MPS_ARCH_I3`, for use internally by the MPS.
There is a design document for the mps interface,
design.mps.interface, but it was written before we had the idea of
having a C interface layer. It is quite relevant, though, and could be
updated. We should use it during the review.
All exported identifiers and file names should begin with ``mps_`` or
:c:macro:`MPS_` so that they don't clash with other systems.
We should probably have a specialized set of rules and a special
checklist for this interface.
:mps:tag:`fmt.extend` This paragraph should be an explanation of why
``mps_fmt_A_t`` is so called. The underlying reason is future
extensibility.
:mps:tag:`thread-safety` Most calls through this interface lock the space
and therefore make the MPM single-threaded. In order to do this they
must recover the space from their parameters. Methods such as
:c:func:`ThreadSpace()` must therefore be callable when the space is *not*
locked. These methods are tagged with the tag of this note.
:mps:tag:`lock-free` Certain functions inside the MPM are thread-safe and do
not need to be serialized by using locks. They are marked with the tag
of this note.
:mps:tag:`form` Almost all functions in this implementation simply cast
their arguments to the equivalent internal types, and cast results
back to the external type, where necessary. Only exceptions are noted
in comments.

407
mps/manual/html/_sources/design/io.txt Normal file
View file

@ -0,0 +1,407 @@
.. _design-io:
.. index::
pair: I/O subsystem; design
I/O subsystem
=============
.. mps:prefix:: design.mps.io
Introduction
------------
:mps:tag:`intro` This document is the design of the MPS I/O Subsystem, a
part of the plinth.
:mps:tag:`readership` This document is intended for MPS developers.
Background
----------
:mps:tag:`bg` This design is partly based on the design of the Internet User
Datagram Protocol (UDP). Mainly I used this to make sure I hadn't left
out anything which we might need.
Purpose
-------
:mps:tag:`purpose` The purpose of the MPS I/O Subsystem is to provide a
means to measure, debug, control, and test a memory manager build
using the MPS.
:mps:tag:`purpose.measure` Measurement consists of emitting data which can
be collected and analysed in order to improve the attributes of
application program, quite possibly by adjusting parameters of the
memory manager (see overview.mps.usage).
:mps:tag:`purpose.control` Control means adjusting the behaviour of the MM
dynamically. For example, one might want to adjust a parameter in
order to observe the effect, then transfer that adjustment to the
client application later.
:mps:tag:`purpose.test` Test output can be used to ensure that the memory
manager is behaving as expected in response to certain inputs.
Requirements
------------
General
.......
:mps:tag:`req.fun.non-hosted` The MPM must be a host-independent system.
:mps:tag:`req.attr.host` It should be easy for the client to set up the MPM
for a particular host (such as a washing machine).
Functional
..........
:mps:tag:`req.fun.measure` The subsystem must allow the MPS to transmit
quantitative measurement data to an external tool so that the system
can be tuned.
:mps:tag:`req.fun.debug` The subsystem must allow the MPS to transmit
qualitative information about its operation to an external tool so
that the system can be debugged.
:mps:tag:`req.fun.control` The subsystem must allow the MPS to receive
control information from an external tool so that the system can be
adjusted while it is running.
:mps:tag:`req.dc.env.no-net` The subsystem should operate in environments
where there is no networking available.
:mps:tag:`req.dc.env.no-fs` The subsystem should operate in environments
where there is no filesystem available.
Architecture
------------
:mps:tag:`arch.diagram` I/O Architecture Diagram
[missing diagram]
:mps:tag:`arch.int` The I/O Interface is a C function call interface by
which the MPM sends and receives "messages" to and from the hosted I/O
module.
:mps:tag:`arch.module` The modules are part of the MPS but not part of the
freestanding core system (see design.mps.exec-env). The I/O module is
responsible for transmitting those messages to the external tools, and
for receiving messages from external tools and passing them to the
MPM.
:mps:tag:`arch.module.example` For example, the "file implementation" might
just send/write telemetry messages into a file so that they can be
received/read later by an off-line measurement tool.
:mps:tag:`arch.external` The I/O Interface is part of interface to the
freestanding core system (see design.mps.exec-env). This is so that
the MPS can be deployed in a freestanding environment, with a special
I/O module. For example, if the MPS is used in a washing machine the
I/O module could communicate by writing output to the seven-segment
display.
Example configurations
......................
:mps:tag:`example.telnet` This shows the I/O subsystem communicating with a
telnet client over a TCP/IP connection. In this case, the I/O
subsystem is translating the I/O Interface into an interactive text
protocol so that the user of the telnet client can talk to the MM.
[missing diagram]
:mps:tag:`example.file` This shows the I/O subsystem dumping measurement
data into a file which is later read and analysed. In this case the
I/O subsystem is simply writing out binary in a format which can be
decoded.
[missing diagram]
:mps:tag:`example.serial` This shows the I/O subsystem communicating with a
graphical analysis tool over a serial link. This could be useful for a
developer who has two machines in close proximity and no networking
support.
:mps:tag:`example.local` In this example the application is talking directly
to the I/O subsystem. This is useful when the application is a
reflective development environment (such as MLWorks) which wants to
observe its own behaviour.
[missing diagram]
Interface
---------
:mps:tag:`if.msg` The I/O interface is oriented around opaque binary
"messages" which the I/O module must pass between the MPM and external
tools. The I/O module need not understand or interpret the contents of
those messages.
:mps:tag:`if.msg.opaque` The messages are opaque in order to minimize the
dependency of the I/O module on the message internals. It should be
possible for clients to implement their own I/O modules for unusual
environments. We do not want to reveal the internal structure of our
data to the clients. Nor do we want to burden them with the details of
our protocols. We'd also like their code to be independent of ours, so
that we can expand or change the protocols without requiring them to
modify their modules.
:mps:tag:`if.msg.dgram` Neither the MPM nor the external tools should assume
that the messages will be delivered in finite time, exactly once, or
in order. This will allow the I/O modules to be implemented using
unreliable transport layers such as the Internet User Datagram Protocl
(UDP). It will also give the I/O module the freedom to drop
information rather than block on a congested network, or stop the
memory manager when the disk is full, or similar events which really
shouldn't cause the memory manager to stop working. The protocols we
need to implement at the high level can be design to be robust against
lossage without much difficulty.
I/O module state
................
:mps:tag:`if.state` The I/O module may have some internal state to preserve.
The I/O Interface defines a type for this state, :c:type:`mps_io_t`, a
pointer to an incomplete structure :c:type:`mps_io_s`. The I/O module is at
liberty to define this structure.
Message types
.............
:mps:tag:`if.type` The I/O module must be able to deliver messages of
several different types. It will probably choose to send them to
different destinations based on their type: telemetry to the
measurement tool, debugging output to the debugger, etc. ::
typedef int mps_io_type_t;
enum {
MPS_IO_TYPE_TELEMETRY,
MPS_IO_TYPE_DEBUG
};
Limits
......
:mps:tag:`if.message-max` The interface will define an unsigned integral
constant :c:macro:`MPS_IO_MESSAGE_MAX` which will be the maximum size of
messages that the MPM will pass to :c:func:`mps_io_send()` (:mps:ref:`.if.send`) and
the maximum size it will expect to receive from :c:func:`mps_io_receive()`.
Interface set-up and tear-down
..............................
:mps:tag:`if.create` The MPM will call :c:func:`mps_io_create()` to set up the I/O
module. On success, this function should return :c:macro:`MPS_RES_OK`. It may
also initialize a "state" value which will be passed to subsequent
calls through the interface.
:mps:tag:`if.destroy` The MPM will call :c:func:`mps_io_destroy()` to tear down
the I/O module, after which it guarantees that the state value will
not be used again. The ``state`` parameter is the state previously
returned by :c:func:`mps_io_create()` (:mps:ref:`.if.create`).
Message send and receive
........................
.. c:function:: extern mps_res_t mps_io_send(mps_io_t state, mps_io_type_t type, void *message, size_t size)
:mps:tag:`if.send` The MPM will call :c:func:`mps_io_send()` when it wishes to
send a message to a destination. The ``state`` parameter is the state
previously returned by :c:func:`mps_io_create()` (:mps:ref:`.if.create`). The
``type`` parameter is the type (:mps:ref:`.if.type`) of the message. The
``message`` parameter is a pointer to a buffer containing the message,
and ``size`` is the length of that message, in bytes. The I/O module
must make an effort to deliver the message to the destination, but is
not expected to guarantee delivery. The function should return
:c:macro:`MPS_RES_IO` only if a serious error occurs that should cause the
MPM to return with an error to the client application. Failure to
deliver the message does not count.
.. note::
Should there be a timeout parameter? What are the timing
constraints? :c:func:`mps_io_send()` shouldn't block.
.. c:function:: extern mps_res_t mps_io_receive(mps_io_t state, void **buffer_o, size_t *size_o)
:mps:tag:`if.receive` The MPM will call :c:func:`mps_io_receive()` when it wants
to see if a message has been sent to it. The ``state`` parameter is
the state previously returned by :c:func:`mps_io_create()` (:mps:ref:`.if.create`).
The ``buffer_o`` parameter is a pointer to a value which should be
updated with a pointer to a buffer containing the message received.
The ``size_o`` parameter is a pointer to a value which should be
updated with the length of the message received. If there is no
message ready for receipt, the length returned should be zero.
.. note::
Should we be able to receive truncated messages? How can this be
done neatly?
I/O module implementations
--------------------------
Routeing
........
The I/O module must decide where to send the various messages. A
file-based implementation could put them in different files based on
their types. A network-based implementation must decide how to address
the messages. In either case, any configuration must either be
statically compiled into the module, or else read from some external
source such as a configuration file.
Notes
-----
The external tools should be able to reconstruct stuff from partial
info. For example, you come across a fragment of an old log containing
just a few old messages. What can you do with it?
Here's some completely untested code which might do the job for UDP.
::
#include "mpsio.h"
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <errno.h>
typedef struct mps_io_s {
int sock;
struct sockaddr_in mine;
struct sockaddr_in telemetry;
struct sockaddr_in debugging;
} mps_io_s;
static mps_bool_t inited = 0;
static mps_io_s state;
mps_res_t mps_io_create(mps_io_t *mps_io_o)
{
int sock, r;
if(inited)
return MPS_RES_LIMIT;
state.mine = /* setup somehow from config */;
state.telemetry = /* setup something from config */;
state.debugging = /* setup something from config */;
/* Make a socket through which to communicate. */
sock = socket(AF_INET, SOCK_DGRAM, 0);
if(sock == -1) return MPS_RES_IO;
/* Set socket to non-blocking mode. */
r = fcntl(sock, F_SETFL, O_NDELAY);
if(r == -1) return MPS_RES_IO;
/* Bind the socket to some UDP port so that we can receive messages. */
r = bind(sock, (struct sockaddr *)&state.mine, sizeof(state.mine));
if(r == -1) return MPS_RES_IO;
state.sock = sock;
inited = 1;
*mps_io_o = &state;
return MPS_RES_OK;
}
void mps_io_destroy(mps_io_t mps_io)
{
assert(mps_io == &state);
assert(inited);
(void)close(state.sock);
inited = 0;
}
mps_res_t mps_io_send(mps_io_t mps_io, mps_type_t type,
void *message, size_t size)
{
struct sockaddr *toaddr;
assert(mps_io == &state);
assert(inited);
switch(type) {
MPS_IO_TYPE_TELEMETRY:
toaddr = (struct sockaddr *)&state.telemetry;
break;
MPS_IO_TYPE_DEBUGGING:
toaddr = (struct sockaddr *)&state.debugging;
break;
default:
assert(0);
return MPS_RES_UNIMPL;
}
(void)sendto(state.sock, message, size, 0, toaddr, sizeof(*toaddr));
return MPS_RES_OK;
}
mps_res_t mps_io_receive(mps_io_t mps_io,
void **message_o, size_t **size_o)
{
int r;
static char buffer[MPS_IO_MESSAGE_MAX];
assert(mps_io == &state);
assert(inited);
r = recvfrom(state.sock, buffer, sizeof(buffer), 0, NULL, NULL);
if(r == -1)
switch(errno) {
/* Ignore interrupted system calls, and failures due to lack */
/* of resources (they might go away.) */
case EINTR: case ENOMEM: case ENOSR:
r = 0;
break;
default:
return MPS_RES_IO;
}
*message_o = buffer;
*size_o = r;
return MPS_RES_OK;
}
Attachments
-----------
"O Architecture Diagram"
"O Configuration Diagrams"

146
mps/manual/html/_sources/design/keyword-arguments.txt
View file

@ -1,6 +1,148 @@
.. _design-keyword-arguments:
.. index::
pair: keyword arguments; design
.. _design-keyword-arguments:
.. include:: ../../converted/keyword-arguments.rst
Keyword arguments in the MPS
============================
Introduction
------------
Up to version 1.111, the `Memory Pool System
<http://www.ravenbrook.com/project/mps/>`_ used varags to pass arguments
to arena and pool classes, because the general MPS interface can't
specify what arguments those classes might need in C prototypes. This
mechanism was error-prone and did not allow for any optional arguments,
meaning that the client had to specify or predict esoteric tuning
parameters.
Starting with version 1.112, the MPS uses an idiom for keyword arguments.
Overview
--------
The basic design is not specific to the MPS. The keyword argument list is
passed as an array of argument structures which look like this::
typedef struct mps_key_s *mps_key_t;
typedef struct mps_arg_s {
mps_key_t key;
union {
int i;
char c;
void *p;
size_t size;
/* etc. */
} val;
} mps_arg_s;
The argument list is assembled and passed like this::
mps_arg_s args[3];
args[0].key = MPS_KEY_MIN_SIZE;
args[0].val.size = 32;
args[1].key = MPS_KEY_MAX_SIZE;
args[1].val.size = 1024;
args[2].key = MPS_KEY_ARGS_END;
mps_pool_create_k(&pool, some_pool_class(), args);
This can be written quite concisely in C99::
mps_pool_create_k(&pool, some_pool_class(),
(mps_arg_s []){{MPS_KEY_MIN_SIZE, {.size = 32}},
{MPS_KEY_MAX_SIZE, {.size = 1024}},
{MPS_KEY_ARGS_END}});
The arguments that are recognised and used by the function are removed
from the array (and the subsequent arguments moved up) so that if they
are all consumed the array has :c:macro:`MPS_KEY_ARGS_END` in slot zero on
return. This can be checked by the caller.
- It's not a static error to pass excess arguments. This makes it easy to
substitute one pool or arena class for another (which might ignore some
arguments). The caller can check that ``args[0].key`` is
:c:macro:`MPS_KEY_ARGS_END` if desired.
- NULL is not a valid argument list. This is in line with general MPS
design principles to avoid accidental omissions. For convenience, we
provide ``mps_args_none`` as a static empty argument list.
- NULL is not a valid argument key. This is in line with general MPS
design principles to avoid accidental omissions. Every key points to
a structure with a signature that can be checked. This makes it virtually
impossible to get an argument list with bad keys or that is unterminated
past MPS checking.
Internals
---------
Internally, keys are static constant structures which are signed and contain
a checking method for the argument, like this::
typedef struct mps_arg_s *Arg;
typedef struct mps_key_s {
Sig sig; /* Always KeySig */
const char *name;
Bool check(Arg arg);
} KeyStruct;
They are mostly declared in the modules that consume them, except for a few
common keys. Declarations look like::
const KeyStruct _mps_key_extend_by = {KeySig, "extend_by", ArgCheckSize};
but ``arg.h`` provides a macro for this::
ARG_DEFINE_KEY(extend_by, Size);
We define keys as static structures (rather than, say, an enum) because:
- The set of keys can be extended indefinitely.
- The set of keys can be extended by indepdently linked modules.
- The structure contents allow strong checking of argument lists.
In the MPS Interface, we declare keys like this::
extern const struct mps_key_s _mps_key_extend_by;
#define MPS_KEY_EXTEND_BY (&_mps_key_extend_by)
The underscore on the symbol requests that client code doesn't reference
it, but instead uses the macro. This gives us adaptability to change the
design and replace keys with, say, magic numbers.
The varargs legacy
------------------
For backward compatibility, varargs to arena and pool creation are
converted into keyword arguments by position, using a method in the
arena or pool class. For example::
static void MVVarargs(ArgStruct args[], va_list varargs)
{
args[0].key = MPS_KEY_EXTEND_BY;
args[0].val.size = va_arg(varargs, Size);
args[1].key = MPS_KEY_MEAN_SIZE;
args[1].val.size = va_arg(varargs, Size);
args[2].key = MPS_KEY_MAX_SIZE;
args[2].val.size = va_arg(varargs, Size);
args[3].key = MPS_KEY_ARGS_END;
AVER(ArgListCheck(args));
}
This leaves the main body of code, and any future code, free to just
handle keyword arguments only.
The use of varargs is deprecated in the manual and the interface and these
methods can be deleted at some point in the future.
References
----------
.. [RB_2012-05-24] Richard Brooksby. Ravenbrook Limited. 2012-05-24. "`Keyword and optional arguments <https://info.ravenbrook.com/mail/2012/05/24/21-19-15/0/>`__".

102
mps/manual/html/_sources/design/lib.txt Normal file
View file

@ -0,0 +1,102 @@
.. _design-lib:
.. index::
pair: library interface; design
Library interface
=================
.. mps:prefix:: design.mps.lib
Introduction
------------
:mps:tag:`intro` This document is the design of the MPS Library Interface, a
part of the plinth.
:mps:tag:`readership` Any MPS developer. Any clients that are prepared to
read this in order to get documentation.
Goals
-----
:mps:tag:`goal` The goals of the MPS library interface are:
:mps:tag:`goal.host` To control the dependency of the MPS on the hosted ISO
C library so that the core MPS remains freestanding (see
design.mps.exec-env).
:mps:tag:`goal.free` To allow the core MPS convenient access to ISO C
functionality that is provided on freestanding platforms (see
design.mps.exec-env.std.com.free).
Description
-----------
Overview
........
:mps:tag:`overview.access` The core MPS needs to access functionality that
could be provided by an ISO C hosted environment.
:mps:tag:`overview.hosted` The core MPS must not make direct use of any
facilities in the hosted environment (design.mps.exec-env). However,
it is sensible to make use of them when the MPS is deployed in a
hosted environment.
:mps:tag:`overview.hosted.indirect` The core MPS does not make any direct
use of hosted ISO C library facilities. Instead, it indirects through
the MPS Library Interface, impl.h.mpslib.
:mps:tag:`overview.free` The core MPS can make direct use of freestanding
ISO C library facilities and does not need to include any of the
header files ``<limits.h>``, ``<stdarg.h>``, and ``<stddef.h>``
directly.
:mps:tag:`overview.complete` The MPS Library Interface can be considered as
the complete "interface to ISO" (in that it provides direct access to
facilities that we get in a freestanding environment and equivalents
of any functionality we require from the hosted environment).
:mps:tag:`overview.provision.client` In a freestanding environment the
client is expected to provide functions meeting this interface to the
MPS.
:mps:tag:`overview.provision.hosted` In a hosted environment,
impl.c.mpsliban may be used. It just maps impl.h.mpslib directly onto
the ISO C library equivalents.
[missing diagram]
Outside the interface
.....................
We provide impl.c.mpsliban to the client, for two reasons:
1. the client can use it to connect the MPS to the ISO C library if it
exists;
2. as an example implementation of the MPS Library Interface.
Implementation
--------------
:mps:tag:`impl` The MPS Library Interface comprises a header file
impl.h.mpslib and some documentation.
:mps:tag:`impl.decl` The header file defines the interface to definitions
which parallel those parts of the non-freestanding ISO headers which
are used by the MPS.
:mps:tag:`impl.include` The header file also includes the freestanding
headers ``<limits.h>``, ``<stdarg.h>``, and ``<stddef.h>`` (and not
``<float.h>``, though perhaps it should).

182
mps/manual/html/_sources/design/lock.txt
View file

@ -1,6 +1,184 @@
.. _design-lock:
.. index::
pair: locking; design
.. _design-lock:
.. include:: ../../converted/lock.rst
The lock module
===============
.. mps:prefix:: design.mps.lock
Purpose
-------
:mps:tag:`purpose` Support the locking needs of the thread-safe design. In
particular:
- recursive locks;
- binary locks;
- recursive "global" lock that need not be allocated or initialized by
the client;
- binary "global" lock that need not be allocated or initialized by
the client.
:mps:tag:`context` The MPS has to be able to operate in a multi-threaded
environment. The thread-safe design (design.mps.thread-safety)
requires client-allocatable binary locks, a global binary lock and a
global recursive lock. An interface to client-allocatable recursive
locks is also present to support any potential use, because of
historic requirements, and because the implementation will presumably
be necessary anyway for the global recursive lock.
Background
----------
:mps:tag:`need` In an environment where multiple threads are accessing
shared data. The threads which access data which is shared with other
threads need to cooperate with those threads to maintain consistency.
Locks provide a simple mechanism for doing this.
:mps:tag:`ownership` A lock is an object which may be "owned" by a single
thread at a time. By claiming ownership of a lock before executing
some piece of code a thread can guarantee that no other thread owns
the lock during execution of that code. If some other thread holds a
claim on a lock, the thread trying to claim the lock will suspend
until the lock is released by the owning thread.
:mps:tag:`data` A simple way of using this behaviour is to associate a lock
with a shared data structure. By claiming that lock around accesses to
the data, a consistent view of the structure can be seen by the
accessing thread. More generally any set of operations which are
required to be mutually exclusive may be performed so by using locks.
Overview
--------
:mps:tag:`adt` There is an abstract datatype :c:type:`Lock` which points to a
locking structure :c:type:`LockStruct`. This structure is opaque to any
client, although an interface is provided to supply the size of the
structure for any client wishing to make a new lock. The lock is not
allocated by the module as allocation itself may require locking.
:c:type:`LockStruct` is implementation specific.
:mps:tag:`simple-lock` There are facilities for claiming and releasing
locks. :c:type:`Lock` is used for both binary and recursive locking.
:mps:tag:`global-locks` "Global" locks are so called because they are used
to protect data in a global location (such as a global variable). The
lock module provides two global locks; one recursive and one binary.
There are facilities for claiming and releasing both of these locks.
These global locks have the advantage that they need not be allocated
or atomically initialized by the client, so they may be used for
locking the initialization of the allocator itself. The binary global
lock is intended to protect mutable data, possibly in conjunction with
other local locking strategies. The recursive global lock is intended
to protect static read-only data during one-off initialization. See
design.mps.thread-safety.
:mps:tag:`deadlock` This module does not provide any deadlock protection.
Clients are responsible for avoiding deadlock by using traditional
strategies such as ordering of locks. (See
design.mps.thread-safety.deadlock.)
:mps:tag:`single-thread` In the single-threaded configuration, locks are not
needed and the claim/release interfaces defined to be no-ops.
Detailed design
---------------
:mps:tag:`interface` The interface comprises the following functions:
.. c:function:: size_t LockSize(void)
Return the size of a :c:type:`LockStruct` for allocation purposes.
.. c:function:: void LockInit(Lock lock)
After initialisation the lock is not owned by any thread.
.. c:function:: void LockFinish(Lock lock)
Before finalisation the lock must not beowned by any thread.
.. c:function:: void LockClaim(Lock lock)
Claims ownership of a lock that was previously not held by current
thread.
.. c:function:: void LockReleaseMPM(Lock lock)
Releases ownership of a lock that is currently owned.
.. c:function:: void LockClaimRecursive(Lock lock)
Remembers the previous state of the lock with respect to the current
thread and claims the lock (if not already held).
.. c:function:: void LockReleaseRecursive(Lock lock)
Testores the previous state of the lock stored by corresponding
:c:func:`LockClaimRecursive()` call.
.. c:function:: void LockClaimGlobal(void)
Claims ownership of the binary global lock which was previously not
held by current thread.
.. c:function:: void LockReleaseGlobal(void)
Releases ownership of the binary global lock that is currently owned.
.. c:function:: void LockClaimGlobalRecursive(void)
Remembers the previous state of the recursive global lock with respect
to the current thread and claims the lock (if not already held).
.. c:function:: void LockReleaseGlobalRecursive(void)
Restores the previous state of the recursive global lock stored by
corresponding :c:func:`LockClaimGlobalRecursive()` call.
:mps:tag:`impl.recursive` For recursive claims, the list of previous states
can be simply implemented by keeping a count of the number of claims
made by the current thread so far. In multi-threaded implementation
below this is handled by the operating system. A count is still kept
and used to check correctness.
:mps:tag:`impl.global` The binary and recursive global locks may actually be
implemented using the same mechanism as normal locks.
:mps:tag:`impl.ansi` Single-Threaded Generic Implementation:
- single-thread;
- no need for locking;
- locking structure contains count;
- provides checking in debug version;
- otherwise does nothing except keep count of claims.
:mps:tag:`impl.win32` Win32 Implementation:
- supports Win32's threads;
- uses Critical Sections [ref?];
- locking structure contains a Critical Section;
- both recursive and non-recursive calls use same Windows function;
- also performs checking.
:mps:tag:`impl.linux` LinuxThreads Implementation (possibly suitable for all
PThreads implementations):
- supports LinuxThreads threads, which are an implementation of
PThreads (see `<http://pauillac.inria.fr/~xleroy/linuxthreads/>`_);
- locking structure contains a mutex, initialized to check for
recursive locking;
- locking structure contains a count of the number of active claims;
- non-recursive locking calls :c:func:`pthread_mutex_lock()` and expects success;
- recursive locking calls :c:func:`pthread_mutex_lock()` and expects either
success or :c:macro:`EDEADLK` (indicating a recursive claim);
- also performs checking.

669
mps/manual/html/_sources/design/locus.txt Normal file
View file

@ -0,0 +1,669 @@
.. _design-locus:
.. index::
pair: locus manager; design
MPS Configuration
=================
.. mps:prefix:: design.mps.locus
Introduction
------------
:mps:tag:`intro` The locus manager coordinates between the pools and takes
the burden of having to be clever about tract/group placement away
from the pools, preserving trace differentiability and contiguity
where appropriate.
:mps:tag:`source` mail.gavinm.1998-02-05.17-52(0), mail.ptw.1998-02-05.19-53(0),
mail.pekka.1998-02-09.13-58(0), and mail.gavinm.1998-02-09.14-05(0).
:mps:tag:`readership` Any MPS developer.
Overview
--------
The MPS manages three main resources:
1. storage;
2. address space;
3. time.
The locus manager manages address space at the arena level.
.. note:
Tucker was right: see mail.ptw.1998-11-02.14-25. Richard Kistruck,
2007-04-24.
When a pool wants some address space, it expresses some preferences to
the locus manager. The locus manager and the arena (working together)
try to honour these preferences, and decide what address space the
pool gets.
Preferences are expressed by the ``SegPref`` argument to
:c:func:`SegAlloc()`. Note that, when they call :c:func:`SegAlloc()`, pools are
asking for address space and writeable storage simultaneously, in a
single call. There is currently no way for pools to reserve address
space without requesting storage.
Why is it important to manage address space?
............................................
1. Trace differentiability
Carefully chosen addresses are used by reference tracing systems
(ie. automatic pools), to categorise objects into clumps; and to
summarise and cheaply find references between clumps.
Different clumps will become worth collecting at different times
(the classic example, of course, is generations in a generational
collector). For these partial collections to be efficient, it must
be cheap to keep these clumps differentiable, cheap to condemn
(Whiten) a particular clump, and cheap to find a good conservative
approximation to all inward references to a clump (both initially
to construct the Grey set, and to make scanning the Grey set
efficient).
This is what the MPS zone mechanism is all about.
The locus manager manages the mapping from clumps to zones.
To specify a clump, pools use the ``SegPrefGen`` argument to
:c:func:`SegPrefExpress()`.
.. note::
The name is misleading: generations are only one sort of clump.
Richard Kistruck, 2007-04-24.
2. Prevent address space fragmentation (within the arena)
Address space is not infinite.
In some use cases, the MPS is required to remain efficient when
using very nearly all available address space and storage. For
example, with the client-arena class, where the only address space
available is that of the storage available.
Even with the VM arena class, typical storage sizes (as of 2007)
can make 32-bit address space constrained: the client may need
several gigabytes, which leaves little spare address space.
Address space fragmentation incurs failure when there is no way to
allocate a big block of address space. The big block may be
requested via the MPS (by the client), or by something else in the
same process, such as a third-party graphics library, image
library, etc.
Address space fragmentation incurs cost when:
- desired large-block requests (such as for buffering) are denied,
causing them to be re-requested as a smaller block, or as several
smaller blocks;
- possible operating-system costs in maintaining a fragmented
mapping?
3. Prevent storage fragmentation (within tracts and segments)
Storage is not infinite: it is allocated in multiples of a
fixed-size tract. Small lonely objects, each retaining a whole
tract, cause storage fragmentation.
Non-moving pools manage this fragmentation with placement
strategies that use:
- co-located death (in space and time);
- segment merging and splitting.
These pool-level strategies always care about contiguity of object
storage. They also often care about the *ordering* of addresses,
because pool code uses an address-ordered search when choosing
where to place a new object. For these two reasons, the address
chosen (by the locus manager and arena) for new tracts is
important.
Certain specialised pools, and/or some client programs that use
them, have carefully tuned segment sizes, positioning, and search
order. Be careful: seemingly inconsequential changes can
catastrophically break this tuning.
Pools can specify a preference for High and Low ends of address
space, which implies a search-order. Pools could also specify
clumping, using either ``SegPrefGen`` or ``SegPrefZoneSet``.
Discovering the layout
......................
The locus manager is not given advance notice of how much address
space will be required with what preferences. Instead, the locus
manager starts with an empty layout, and adapts it as more requests
come in over time. It is attempting to discover a suitable layout by
successive refinement. This is ambitious.
Definitions
-----------
:mps:tag:`note.cohort` We use the word "cohort" in its usual sense here, but
we're particularly interested in cohorts that have properties relevant
to tract placement. It is such cohorts that the pools will try to
organize using the services of the locus manager. Typical properties
would be trace differentiability or (en masse) death-time
predictability. Typical cohorts would be instances of a
non-generational pool, or generations of a collection strategy.
:mps:tag:`def.trace.differentiability` Objects (and hence tracts) that are
collected, may or may not have "trace differentiability" from each
other, depending on their placement in the different zones. Objects
(or pointers to them) can also have trace differentiability (or not)
from non-pointers in ambiguous references; in practice, we will be
worried about low integers, that may appear to be in zones 0 or -1.
Requirements
------------
:mps:tag:`req.cohort` Tract allocations must specify the cohort they
allocate in. These kind of cohorts will be called loci, and they will
have such attributes as are implied by the other requirements.
Critical.
:mps:tag:`req.counter.objects` As a counter-requirement, pools are expected
to manage objects. Objects the size of a tract allocation request
(segment-sized) are exceptional. Critical.
:mps:tag:`req.counter.objects.just` This means the locus manager is not
meant to solve the problems of allocating large objects, and it isn't
required to know what goes on in pools.
:mps:tag:`req.contiguity` Must support a high level of contiguity within
cohorts when requested. This means minimizing the number of times a
cohort is made aware of discontiguity. Essential (as we've effectively
renegotiated this in SW, down to a vague hope that certain critical
cohorts are not too badly fragmented). :mps:tag:`req.contiguity.just` TSBA.
:mps:tag:`req.contiguity.specific` It should be possible to request another
allocation next to a specific tract on either side (or an extension in
that direction, as the case may be). Such a request can fail, if
there's no space there. Nice. It would also be nice to have one for
"next to the largest free block".
:mps:tag:`req.differentiable` Must support the trace differentiability of
segments that may be condemned separately. Due to the limited number
of zones, it must be possible to place several cohorts into the same
zone. Essential.
:mps:tag:`req.differentiable.integer` It must be possible to place
collectable allocations so that they are trace-differentiable from
small integers. Essential.
:mps:tag:`req.disjoint` Must support the disjointness of pages that have
different VM properties (such as mutable/immutable,
read-only/read-write, and different lifetimes). Optional.
.. note::
I expect the implementation will simply work at page or larger
granularity, so the problem will not arise, but Tucker insisted on
stating this as a requirement. Pekka P. Pirinen, 1998-10-28.
:mps:tag:`req.low-memory` The architecture of the locus manager must not
prevent the design of efficient applications that often use all
available memory. Critical. :mps:tag:`req.low-memory.expl` This basically
says it must be designed to perform well in low-memory conditions, but
that there can be configurations where it doesn't do as well, as long
as this is documented for the application programmer. Note that it
doesn't say all applications are efficient, only that if you manage to
design an otherwise efficient application, the locus manager will not
sink it.
:mps:tag:`req.address` Must conserve address space in VM arenas to a
reasonable extent. Critical.
:mps:tag:`req.inter-pool` Must support the association of sets of tracts in
different pools into one cohort. Nice.
:mps:tag:`req.ep-style` Must support the existing EP-style of allocation
whereby allocation is from one end of address space either upwards or
downwards (or a close approximation thereto with the same behavior).
:mps:tag:`req.ep-style.just` We cannot risk disrupting a policy with
well-known properties when this technology is introduced.
:mps:tag:`req.attributes` There should be a way to inform the locus manager
about various attributes of cohorts that might be useful for
placement: deathtime, expected total size, and so on. Optional. It's a
given that the cohorts must then have these attributes, within the
limits set in the contract of the appropriate interface.
:mps:tag:`req.attributes.action` The locus manager should use the attributes
to guide its placement decisions. Nice.
:mps:tag:`req.blacklisting` There should be a way of maintaining at least
one blacklist for pages (or some other small unit), that can
not/should not be allocated to collectable pools. Optional.
.. note::
How to do blacklist breaking for ambiguous refs?
:mps:tag:`req.hysteresis` There should be a way to indicate which cohorts
fluctuate in size and by how much, to guide the arena hysteresis to
hold on to suitable pages. Optional.
Analysis
--------
:mps:tag:`anal.sw` Almost any placement policy would be an improvement on
the current SW one.
:mps:tag:`anal.cause-and-effect` The locus manager doesn't usually need to
know *why* things need to be differentiable, disjoint, contiguous, and
so on. Abstracting the reason away from the interface makes it more
generic, more likely to have serendipitous new uses. Attributes
described by a quantity (deathtime, size, etc.) are an exception to
this, because we can't devise a common measure.
:mps:tag:`anal.stable` The strategy must be stable: it must avoid repeated
recomputation, especially the kind that switches between alternatives
with a short period (repeated "bites" out the same region or
flip-flopping between two regions).
:mps:tag:`anal.fragmentation` There's some call to avoid fragmentation in
cohorts that don't need strict contiguity, but this is not a separate
requirement, since fragmentation is a global condition, and can only
be ameliorated if there's a global strategy that clumps allocations
together.
:mps:tag:`anal.deathtime` Cohorts with good death-time clumping of their
objects could use some locality of tract allocation, because it
increases the chances of creating large holes in the address space
(for other allocation to use). OTOH. many cohorts will not do multiple
frees in short succession, or at least cannot reasonably be predicted
to do so. This locality is not contiguity, nor is it low
fragmentation, it's just the requirement to place the new tracts next
to the tract where the last object was allocated in the cohort. Note
that the placement of objects is under the control of the pool, and
the locus manager will not know it, therefore this requirement should
be pursued by requesting allocation next to a particular tract (which
we already have a requirement for).
:mps:tag:`anal.asymmetrical` The strategy has to be asymmetrical with
respect to cohorts growing and shrinking. The reason of this asymmetry
is that it can choose where to grow, but it cannot choose where to
shrink (except in a small way by growing with good locality).
Interface
---------
:mps:tag:`interface.locus` A cohort will typically reside on multiple tracts
(and the pools will avoid putting objects of other cohorts on them),
so there should be an interface to describe the properties of the
cohort, and associate each allocation request with the cohort. We
shall call such an object, created to represent a cohort, a locus (pl.
loci).
:mps:tag:`interface.locus.pool` Loci will usually be created by the pool
that uses it. Some of the locus attributes will be inherited from
client-specified pool attributes [this means there will be additional
pool attributes].
:mps:tag:`interface.detail` This describes interface in overview; for
details, see implementation section and code, or user doc.
Loci
....
.. c:function:: Res LocusCreate(Locus *locusReturn, LocusAttrs attrs, ZoneGroup zg, LocusAllocDesc adesc)
:mps:tag:`function.create` A function to create a locus: ``adesc`` contains
the information about the allocation sequences in the locus, ``zg`` is
used for zone differentiability, and ``attrs`` encodes the following:
- :mps:tag:`locus.contiguity` A locus can be contiguous. This means
performing as required in :mps:ref:`.req.contiguity`, non-contiguous
allocations can be freely placed anywhere (but efficiency dictates
that similar allocations are placed close together and apart from
others).
- :mps:tag:`locus.blacklist` Allocations in the locus will avoid blacklisted
pages (for collectable segments).
- :mps:tag:`locus.zero` Allocations in the locus are zero-filled.
.. note::
Other attributes will be added, I'm sure.
:mps:tag:`interface.zone-group` The locus can be made a member of a zone
group. Passing ``ZoneGroupNONE`` means it's not a member of any group
(allocations will be placed without regard to zone, except to keep
them out of stripes likely to be needed for some group).
.. note::
I propose no mechanism for managing zone groups at this time,
since it's only used internally for one purpose. Pekka P. Pirinen,
2000-01-17.
:mps:tag:`interface.size` An allocation descriptor (``LocusAllocDesc``)
contains various descriptions of how the locus will develop over time
(inconsistent specifications are forbidden, of course):
- :mps:tag:`interface.size.typical-alloc` Size of a typical allocation in
this locus, in bytes. This will mainly affect the grouping of
non-contiguous loci.
- :mps:tag:`interface.size.large-alloc` Typical large allocation that the
manager should try to allow for (this allows some relief from
:mps:ref:`.req.counter.objects`), in bytes. This will mainly affect the size
of gaps that will be allotted adjoining this locus.
- :mps:tag:`interface.size.direction` Direction of growth: up/down/none.
Only useful if the locus is contiguous.
- :mps:tag:`interface.size.lifetime` Some measure of the lifetime of tracts
(not objects) in the cohort.
.. note::
Don't know the details yet, probably only useful for placing
similar cohorts next to each other, so the details don't
actually matter. Pekka P. Pirinen, 2000-01-17.
- :mps:tag:`interface.size.deathtime` Some measure of the deathtime of
tracts (not objects) in the cohort.
.. note::
Ditto. Pekka P. Pirinen, 2000-01-17.
:mps:tag:`function.init` :c:func:`LocusInit()` is like :c:func:`LocusCreate()`, but
without the allocation. This is the usual interface, since most loci
are embedded in a pool or something.
:mps:tag:`function.alloc` :c:func:`ArenaAlloc()` to take a locus argument.
:c:func:`ArenaAllocHere()` is like it, plus it takes a tract and a
specification to place the new allocation immediately above/below a
given tract; if that is not possible, it returns ``ResFAIL`` (this
will make it useful for reallocation functionality).
.. c:function:: ArenaSetTotalLoci(Arena arena, Size nLoci, Size nZoneGroups)
:mps:tag:`function.set-total` A function to tell the arena the expected
number of (non-miscible client) loci, and of zone groups.
Peaks
.....
.. c:function:: mps_res_t mps_peak_create(mps_peak_t*, mps_arena_t)
:mps:tag:`function.peak.create` A function to create a peak. A newly-created
peak is open, and will not be used to guide the strategy of the locus
manager.
.. c:function:: mps_res_t mps_peak_describe_pool(mps_peak_t, mps_pool_t, mps_size_desc_t)
:mps:tag:`function.peak.add` A function to add a description of the state of
one pool into the peak. Calling this function again for the same peak and pool instance will replace
the earlier description.
:mps:tag:`function.peak.add.size` The size descriptor contains a total size
in bytes or percent of arena size.
.. note::
Is this right? Pekka P. Pirinen, 2000-01-17.
:mps:tag:`function.peak.add.remove` Specifying a :c:macro:`NULL` size will remove
the pool from the peak. The client is not allowed to destroy a pool
that is mentioned in any peak; it must be first removed from the peak,
or the peak must be destroyed. This is to ensure that the client
adjusts the peaks in a manner that makes sense to the application; the
locus manager can't know how to do that.
.. c:function:: mps_res_t mps_peak_close(mps_peak_t)
:mps:tag:`function.peak.close` A function to indicate that all the
significant pools have been added to the peak, and it can now be used
to guide the locus manager. For any pool not described in the peak,
the locus manager will take its current size at any given moment as
the best prediction of its size at the peak.
:mps:tag:`function.peak.close.after` It is legal to add more descriptions to
the peak after closing, but this will reopen the peak, and it will
have to be closed before the locus manager will use it again. The
locus manager uses the previous closed state of the peak, while this
is going on.
.. c:function:: void mps_peak_destroy(mps_peak_t)
:mps:tag:`function.peak.destroy` A function to destroy a peak.
:mps:tag:`interface.ep-style` This satisfies :mps:ref:`.req.ep-style` by allowing SW
to specify zero size for most pools (which will cause them to be place
next to other loci with the same growth direction).
.. note::
Not sure this is good enough, but we'll try it first. Pekka P.
Pirinen, 2000-01-17.
Architecture
------------
Data objects
............
:mps:tag:`arch.locus` To represent the cohorts, we have locus objects.
Usually a locus is embedded in a pool instance, but generations are
separate loci.
:mps:tag:`arch.locus.attr` contiguity, blacklist, zg, current region, @@@@
:mps:tag:`arch.locus.attr.exceptional` The client can define a typical large
allocation for the locus. Requests substantially larger than that are
deemed exceptional.
:mps:tag:`arch.zone-group` To satisfy :mps:ref:`.req.condemn`, we offer zone groups.
Each locus can be a member of a zone group, and the locus manager will
attempt to place allocations in this locus in different zones from all
the other zone groups. A zone-group is represented as @@@@.
:mps:tag:`arch.page-table` A page table is maintained by the arena, as usual
to track association between tracts, pools and segments, and mapping
status for VM arenas.
:mps:tag:`arch.region` All of the address space is divided into disjoint
regions, represented by region objects. These objects store their
current limits, and high and low watermarks of currently allocated
tracts (we hope there's usually a gap of empty space between regions).
The limits are actually quite porous and flexible.
:mps:tag:`arch.region.assoc` Each region is associated with one contiguous
locus or any number of non-contiguous loci (or none). We call the
first kind of region "contiguous". :mps:tag:`arch.locus.assoc` Each locus
remembers all regions where it has tracts currently, excepting the
badly-placed allocations (see below). It is not our intention that any
locus would have very many, or that loci that share regions would have
any reason to stop doing do.
:mps:tag:`arch.region.more` Various quantities used by the placement
computation are also stored in the regions and the loci. Regions are
created (and destroyed) by the placement recomputation. Regions are
located in stripes (if it's a zoned region), but they can extend into
neighboring stripes if an exceptionally large tract allocation is
requested (to allow for large objects).
:mps:tag:`arch.chunk` Arenas may allocate more address space in additional
chunks, which may be disjoint from the existing chunks. Inter-chunk
space will be represented by dummy regions. There are also sentinel
regions at both ends of the address space.
Overview of strategy
....................
:mps:tag:`arch.strategy.delay` The general strategy is to delay placement
decisions until they have to be made, but no later.
:mps:tag:`arch.strategy.delay.until` Hence, the locus manager only makes
placement decisions when an allocation is requested (frees and other
operations might set a flag to cause the next allocation to redecide).
This also allows the client to change the peak and pool configuration
in complicated ways without causing a lot of recomputation, by doing
all the changes without allocating in the middle (unless the control
pool needs more space because of the changes).
:mps:tag:`arch.strategy.normal` While we want the placement to be
sophisticated, we do not believe it is worth the effort to consider
all the data at each allocation. Hence, allocations are usually just
placed in one of the regions used previously (see :mps:ref:`.arch.alloc`)
without reconsidering the issues.
:mps:tag:`arch.strategy.normal.limit` However, the manager sets
precautionary limits on the regions to ensure that the placement
decisions are revisited when an irrevocable placement is about to be
made.
:mps:tag:`arch.strategy.create` The manager doesn't create new regions until
they are needed for allocation (but it might compute where they could
be placed to accommodate a peak).
Allocation
..........
:mps:tag:`arch.alloc` Normally, each allocation to a locus is placed in its
current region. New regions are only sought when necessary to fulfill
an allocation request or when there is reason to think the situation
has changed significantly (see :mps:ref:`.arch.significant`).
:mps:tag:`arch.alloc.same` An allocation is first attempted next to the
previous allocation in the same locus, respecting growth direction. If
that is not possible, a good place in the current region is sought.
:mps:tag:`arch.alloc.same.hole` At the moment, for finding a good place
within a region, we just use the current algorithm, limited to the
region. In future, the placement within regions will be more clever.
:mps:tag:`arch.alloc.extend` If there's no adequate hole in the current
region and the request is not exceptional, the neighboring regions are
examined to see if the region could be extended at one border. (This
will basically only be done if the neighbor has shrunk since the last
placement recomputation, because the limit was set on sophisticated
criteria, and should not be changed without justification.)
:mps:tag:`arch.alloc.extend.here` When an allocation is requested next to a
specific tract (:c:func:`ArenaAllocHere()`), we try to extend a little
harder (at least for ``change_size``, perhaps not for locality).
:mps:tag:`arch.alloc.other` If no way can be found to allocate in the
current region, other regions used for this locus are considered in
the same way, to see if space can be found there. [Or probably look at
other regions before trying to extend anything?]
:mps:tag:`arch.alloc.recompute` When no region of this locus has enough
space for the request, or when otherwise required, region placement is
recomputed to find a new region for the request (which might be the
same region, after extension).
:mps:tag:`arch.alloc.current` This region where the allocation was placed
then becomes the current region for this locus, except when the
request was exceptional, or when the region chosen was "bad" (see
@@@@).
:mps:tag:`arch.significant` Significant changes to the parameters affecting
placement are deemed to have happened at certain client calls and when
the total allocation has changed substantially since the last
recomputation. Such conditions set a flag that causes the next
allocation to recompute even if its current region is not full
(possibly second-guess the decision to recompute after some
investigation of the current state?).
Deallocation
............
:mps:tag:`arch.free` Deallocation simply updates the counters in the region
and the locus. For some loci, it will make the region of the
deallocation the current region. :mps:tag:`arch.free.remove` If a region
becomes entirely empty, it is deleted (and the neighbors limits might
be adjusted).
.. note::
This is quite tricky to get right.
Region placement recomputation
..............................
:mps:tag:`arch.gap` When doing placement computations, we view the arena as
a sequence of alternating region cores and gaps (which can be small,
even zero-sized). Initially, we'll take the core of a region to be the
area between the high and low watermark, but in the future we might be
more flexible about that.
.. note::
Edge determination is actually a worthwhile direction to explore.
:mps:tag:`arch.reach` The gap between two cores could potentially end up
being allocated to either region, if they grow in that direction, or
one or neither, if they don't. The set of states that the region
assignment could reach by assigning the gaps to their neighbors is
called the reach of the current configuration.
:mps:tag:`arch.placement.object` The object of the recomputation is to find
a configuration of regions that is not too far from the current
configuration and that keeps all the peaks inside its reach; if that
is not possible, keep the nearest ones in the reach and then minimize
the total distance from the rest.
:mps:tag:`arch.placement.hypothetical` The configurations that are
considered will include hypothetical placements for new regions for
loci that cannot fit in their existing regions at the peak. This is
necessary to avoid choosing a bad alternative.
:mps:tag:`arch.placement.interesting` The computation will only consider new
regions of loci that are deemed interesting, that is, far from their
peak state. This will reduce the computational burden and avoid
jittering near a peak.
.. note::
Details missing.
Implementation
--------------
[missing]
Notes
-----
:mps:tag:`idea.change` Even after the first segment, be prepared to change
your mind, if by the second segment a lot of new loci have been
created.
:mps:tag:`distance` If the current state is far from a peak, there's time to
reassign regions and for free space to appear (in fact, under the
steady arena assumption, enough free space *will* appear).
:mps:tag:`clear-pool` Need to have a function to deallocate all objects in a
pool, so that :c:func:`PoolDestroy()` won't have to be used for that
purpose.

292
mps/manual/html/_sources/design/message-gc.txt Normal file
View file

@ -0,0 +1,292 @@
.. _design-message-gc:
.. index::
pair: garbage collection messages; design
GC messages
===========
.. mps:prefix:: design.mps.config
Introduction
------------
:mps:tag:`intro` This document describes the design of the MPS garbage
collection messages. For a guide to the MPS message system in general,
see design.mps.message.
:mps:tag:`readership` Any MPS developer.
Overview
--------
The MPS provides two types of GC messages:
- :c:func:`mps_message_type_gc_start()`;
- :c:func:`mps_message_type_gc()`.
They are called "trace start" and "trace end" messages in this
document and in most MPS source code.
Introduction
------------
The MPS posts a trace start message (:c:func:`mps_message_type_gc_start()`)
near the start of every trace (but after calculating the condemned
set, so we can report how large it is).
The MPS posts a trace end message (:c:func:`mps_message_type_gc()`) near the
end of every trace.
These messages are extremely flexible: they can hold arbitrary
additional data simply by writing new accessor functions. If there is
more data to report at either of these two events, then there is a
good argument for adding it into these existing messages.
.. note::
In previous versions of this design document, there was a partial
unimplemented design for an :c:func:`mps_message_type_gc_generation()`
message. This would not have been a good design, because managing
and collating multiple messages is much more complex for both MPS
and client than using a single message. Richard Kistruck,
2008-12-19.
Purpose
-------
:mps:tag:`purpose` The purpose of these messages is to allow the client
program to be aware of GC activity, in order to:
- adjust its own behaviour programmatically;
- show or report GC activity in a custom way, such as an in-client
display, in a log file, etc.
The main message content should be intelligible and helpful to
client-developers (with help from MPS staff if necessary). There may
be extra content that is only meaningful to MPS staff, to help us
diagnose client problems.
While there is some overlap with the Diagnostic Feedback system
(design.mps.diag), the main contrasts are that these GC messages are
present in release builds, are stable from release to release, and are
designed to be parsed by the client program.
Names and parts
---------------
Here's a helpful list of the names used in the GC message system:
Implementation is mostly in the source file ``traceanc.c`` (trace
ancillary).
============================= ============================== ======================
Internal name "trace start" "trace end"
Internal type ``TraceStartMessage`` ``TraceMessage``
:c:type:`ArenaStruct` member ``tsMessage[]`` ``tMessage``
Message type ``MessageTypeGCSTART`` ``MessageTypeGC``
External name ``mps_message_type_gc_start`` ``mps_message_type_gc``
============================= ============================== ======================
.. note::
The names of these messages are unconventional; they should
properly be called "gc (or trace) *begin*" and "gc (or trace)
*end*". But it's much too late to change them now. Richard
Kistruck, 2008-12-15.
Collectively, the trace-start and trace-end messages are called the
"trace id messages", and they are managed by the functions
:c:func:`TraceIdMessagesCheck()`, :c:func:`TraceIdMessagesCreate()`, and :c:func:`TraceIdMessagesDestroy()`.
The currently supported message-field accessor methods are:
:c:func:`mps_message_gc_start_why()`, :c:func:`mps_message_gc_live_size()`,
:c:func:`mps_message_gc_condemned_size()`, and
:c:func:`mps_message_gc_not_condemned_size()`. These are documented in the
Reference Manual.
Lifecycle
---------
:mps:tag:`lifecycle` for each trace id, pre-allocate a pair of start/end
messages by calling :c:func:`ControlAlloc()`. Then, when a trace runs using
that trace id, fill in and post these messages. As soon as the trace
has posted both messages, immediately pre-allocate a new pair of
messages, which wait in readiness for the next trace to use that
trace id.
Requirements
............
:mps:tag:`req.no-start-alloc` Should avoid attempting to allocate memory at
trace start time. :mps:tag:`req.no-start-alloc.why` There may be no free
memory at trace start time. Client would still like to hear about
collections in those circumstances.
:mps:tag:`req.queue` Must support a client that enables, but does not
promptly retrieve, GC messages. Messages that have not yet been
retrived must remain queued, and the client must be able to retrieve
them later without loss. It is not acceptable to stop issuing GC
messages for subsequent collections merely because messages from
previous collections have not yet been retrieved. :mps:tag:`req.queue.why`
This is because there is simply no reasonable way for a client to
guarantee that it always promptly collects GC messages.
:mps:tag:`req.match` Start and end messages should always match up: never
post one of the messages but fail to post the matching one.
:mps:tag:`req.match.why` This makes client code much simpler -- it does not
have to handle mismatching messages.
:mps:tag:`req.errors-not-direct` Errors (such as a :c:func:`ControlAlloc()`
failure) cannot be reported directly to the client, because
collections often happen automatically, without an explicit client
call to the MPS interface.
:mps:tag:`req.multi-trace` Up to ``TraceLIMIT`` traces may be running, and
emitting start/end messages, simultaneously.
:mps:tag:`req.early` Nice to tell client as much as possible about the
collection in the start message, if we can.
:mps:tag:`req.similar` Start and end messages are conceptually similar -- it
is quite okay, and may be helpful to the client, for the same datum
(for example: the reason why the collection occurred) to be present in
both the start and end message.
Storage
.......
For each trace-id (:mps:ref:`.req.multi-trace`) a pair (:mps:ref:`.req.match`) of
start/end messages is dynamically allocated (:mps:ref:`.req.queue`) in advance
(:mps:ref:`.req.no-start-alloc`). Messages are allocated in the control pool
using :c:func:`ControlAlloc()`.
.. note::
Previous implementations of the trace start message used static
allocation. This does not satisfy :mps:ref:`.req.queue`. See also
job001570_. Richard Kistruck, 2008-12-15.
.. _job001570: http://www.ravenbrook.com/project/mps/issue/job001570/
Poiters to these messages are stored in ``tsMessage[ti]`` and
``tMessage[ti]`` arrays in the :c:type:`ArenaStruct`.
.. note::
We must not> keep the pre-allocated messages, or pointers to them,
in :c:type:`TraceStruct`: the memory for these structures is statically
allocated, but the values in them are re-initialised by
:c:func:`TraceCreate()` each time the trace id is used, so the
:c:func:`TraceStruct()` is invalid (that is: to be treated as random
uninitialised memory) when not being used by a trace. See also
job001989_. Richard Kistruck, 2008-12-15.
.. _job001989: http://www.ravenbrook.com/project/mps/issue/job001989/
Creating and Posting
....................
In :c:func:`ArenaCreate()` we use :c:macro:`TRACE_SET_ITER` to initialise the
``tsMessage[ti]`` and ``tMessage[ti]`` pointers to :c:macro:`NULL`, and then
(when the control pool is ready) :c:macro:`TRACE_SET_ITER` calling
:c:func:`TraceIdMessagesCreate()`. This performs the initial pre-allocation
of the trace start/end messages for each trace id. Allocation failure
is not tolerated here: it makes :c:func:`ArenaCreate()` fail with an error
code, because the arena is deemed to be unreasonably small.
When a trace is running using trace id ``ti``, it finds a
pre-allocated message via ``tsMessage[ti]`` or ``tMessage[ti]`` in the
:c:func:`ArenaStruct()`, fills in and posts the message, and nulls-out the
pointer. (If the pointer was null, no message is sent; see below.) The
message is now reachable only from the arena message queue (but the
control pool also knows about it).
When the trace completes, it calls :c:func:`TraceIdMessagesCreate()` for its
trace id. This performs the ongoing pre-allocation of the trace
start/end messages for the next use of this trace id. The expectation
is that, after a trace has completed, some memory will have been
reclaimed, and the :c:func:`ControlAlloc()` will succeed.
But allocation failure here is permitted: if it happens, both the
start and the end messages are freed (if present). This means that,
for the next collection using this trace id, neither a start nor an
end message will be sent (:mps:ref:`.req.match`). There is no direct way to
report this failure to the client (:mps:ref:`.req.errors-not-direct`), so we
just increment the ``droppedMessages`` counter in the :c:type:`ArenaStruct`.
Currently this counter is never reported to the client (except in
diagnostic varieties).
Getting and discarding
......................
If the client has not enabled that message type, the message is
discarded immediately when posted, calling :c:func:`ControlFree()` and
reclaiming the memory.
If the client has enabled but never gets the message, it remains on
the message queue until :c:func:`ArenaDestroy()`. Theoretically these
messages could accumulate forever until they exhaust memory. This is
intentional: the client should not enable a message type and then
never get it!
Otherwise, when the client gets a message, it is dropped from the
arena message queue: now only the client (and the control pool) hold
references to it. The client must call :c:func:`mps_message_discard()` once
it has finished using the message. This calls :c:func:`ControlFree()` and
reclaims the memory.
If the client simply drops its reference, the memory will not be
reclaimed until :c:func:`ArenaDestroy()`. This is intentional: the control
pool is not garbage-collected.
Final clearup
.............
Final clearup is performed at :c:func:`ArenaDestroy()`, as follows:
- Unused and unsent pre-allocated messages (one per trace id) are
freed with :c:macro:`TRACE_SET_ITER` calling :c:func:`TraceIdMessagesDestroy()`
which calls the message Delete functions (and thereby
:c:func:`ControlFree()`) on anything left in ``tsMessage[ti]`` and
``tMessage[ti]``.
- Unretrieved messages are freed by emptying the arena message queue
with :c:func:`MessageEmpty()`.
- Retrieved but undiscarded messages are freed by destroying the
control pool.
Testing
-------
The main test is "``zmess.c``". See notes there.
Various other tests, including ``amcss.c``, also collect and report
:c:func:`mps_message_type_gc()` and :c:func:`mps_message_type_gc_start()`.
Coverage
........
Current tests do not check:
- The less common why-codes (reasons why a trace starts). These should
be added to ``zmess.c``.

371
mps/manual/html/_sources/design/message.txt
View file

@ -1,7 +1,374 @@
.. _design-message:
.. index::
pair: messages; design
single: client message protocol
.. _design-message:
.. include:: ../../converted/message.rst
Client message protocol
=======================
.. mps:prefix:: design.mps.message
pair: messages; design
single: client message protocol
Introduction
------------
:mps:tag:`intro` The client message protocol provides a means by which
clients can receive messages from the MPS asynchronously. Typical
messages may be low memory notification (or in general low utility),
finalization notification, soft-failure notification. There is a
general assumption that it should not be disastrous for the MPS client
to ignore messages, but that it is probably in the clients best
interest to not ignore messages. The justification for this is that
the MPS cannot force the MPS client to read and act on messages, so no
message should be critical.
.. note::
Bogus, since we cannot force clients to check error codes either.
Pekka P. Pirinen, 1997-09-17.
:mps:tag:`contents` This document describes the design of the external and
internal interfaces and concludes with a sketch of an example design
of an internal client. The example is that of implementing
finalization using PoolMRG.
:mps:tag:`readership` Any MPS developer.
Requirements
------------
:mps:tag:`req` The client message protocol will be used for implementing
finalization (see design.mps.finalize and req.dylan.fun.final). It
will also be used for implementing the notification of various
conditions (possibly req.dylan.prot.consult is relevant here).
External interface
------------------
:mps:tag:`if.queue` Messages are presented as a single queue per arena.
Various functions are provided to inspect the queue and inspect
messages in it (see below).
Functions
.........
:mps:tag:`if.fun` The following functions are provided:
:mps:tag:`if.fun.poll` :c:func:`mps_message_poll()` sees whether there are any
messages pending. Returns 1 only if there is a message on the queue of
arena. Returns 0 otherwise.
:mps:tag:`if.fun.enable` :c:func:`mps_message_type_enable()` enables the flow of
messages of a certain type. The queue of messages of a arena will
contain only messages whose types have been enabled. Initially all
message types are disabled. Effectively this function allows the
client to declare to the MPS what message types the client
understands. The MPS does not generate any messages of a type that
hasn't been enabled. This allows the MPS to add new message types (in
subsequent releases of a memory manager) without confusing the client.
The client will only be receiving the messages if they have explicitly
enabled them (and the client presumably only enables message types
when they have written the code to handle them).
:mps:tag:`if.fun.disable` :c:func:`mps_message_type_disable()` disables the flow
of messages of a certain type. The antidote to
:c:func:`mps_message_type_enable()`. Disables the specified message type.
Flushes any existing messages of that type on the queue, and stops any
further generation of messages of that type. This permits clients to
dynamically decline interest in a message type, which may help to
avoid a memory leak or bloated queue when the messages are only
required temporarily.
:mps:tag:`if.fun.get` :c:func:`mps_message_get()` begins a message "transaction".
If there is a message of the specified type on the queue then the
first such message will be removed from the queue and a handle to it
will be returned to the client via the ``messageReturn`` argument; in
this case the function will return :c:macro:`TRUE`. Otherwise it will return
:c:macro:`FALSE`. Having obtained a handle on a message in this way, the
client can use the type-specific accessors to find out about the
message. When the client is done with the message the client should
call :c:func:`mps_message_discard()`; failure to do so will result in a
resource leak.
:mps:tag:`if.fun.discard` :c:func:`mps_message_discard()` ends a message
"transaction". It indicates to the MPS that the client is done with
this message and its resources may be reclaimed.
:mps:tag:`if.fun.type.any` :c:func:`mps_message_queue_type()` determines the type
of a message in the queue. Returns :c:macro:`TRUE` only if there is a message
on the queue of arena, and in this case updates the ``typeReturn``
argument to be the type of a message in the queue. Otherwise returns
:c:macro:`FALSE`.
:mps:tag:`if.fun.type` :c:func:`mps_message_type()` determines the type of a
message (that has already been got). Only legal when inside a message
transaction (that is, after :c:func:`mps_message_get()` and before
:c:func:`mps_message_discard()`). Note that the type will be the same as the
type that the client passed in the call to :c:func:`mps_message_get()`.
Types of messages
.................
:mps:tag:`type` The type governs the "shape" and meaning of the message.
:mps:tag:`type.int` Types themselves will just be a scalar quantity, an
integer.
:mps:tag:`type.semantics` A type indicates the semantics of the message.
:mps:tag:`type.semantics.interpret` The semantics of a message are
interpreted by the client by calling various accessor methods on the
message.
:mps:tag:`type.accessor` The type of a message governs which accessor
methods are legal to apply to the message.
:mps:tag:`type.example` Some example types:
:mps:tag:`type.finalization` There will be a finalization type. The type is
abstractly: ``FinalizationMessage(Ref)``.
:mps:tag:`type.finalization.semantics` A finalization message indicates that
an object has been discovered to be finalizable (see
design.mps.poolmrg.def.final.object for a definition of finalizable).
:mps:tag:`type.finalization.ref` There is an accessor to get the reference
of the finalization message (i.e. a reference to the object which is
finalizable) called :c:func:`mps_message_finalization_ref()`.
:mps:tag:`type.finalization.ref.scan` Note that the reference returned
should be stored in scanned memory.
Compatibility issues
....................
:mps:tag:`compatibility` The following issues affect future compatibility of
the interface:
:mps:tag:`compatibility.future.type-new` Notice that message of a type that
the client doesn't understand are not placed on the queue, therefore
the MPS can introduce new types of message and existing client will
still function and will not leak resources. This has been achieved by
getting the client to declare the types that the client understands
(with :c:func:`mps_message_type_enable()`, :mps:ref:`.if.fun.enable`).
:mps:tag:`compatibility.future.type-extend` The information available in a
message of a given type can be extended by providing more accessor
methods. Old clients won't get any of this information but that's
okay.
Internal interface
------------------
Types
.....
.. c:type:: struct MessageStruct *Message
:mps:tag:`message.type` :c:type:`Message` is the type of messages.
:mps:tag:`message.instance` Messages are instances of Message Classes.
.. c:type:: struct MessageStruct *MessageStruct
:mps:tag:`message.concrete` Concretely a message is represented by a
:c:type:`MessageStruct`. A :c:type:`MessageStruct` has the usual signature field
(see design.mps.sig). A :c:type:`MessageStruct` has a type field which
defines its type, a ring node, which is used to attach the message to
the queue of pending messages, a class field, which identifies a
:c:type:`MessageClass` object.
:mps:tag:`message.intent` The intention is that a :c:type:`MessageStruct` will be
embedded in some richer object which contains information relevant to
that specific type of message.
:mps:tag:`message.struct` The structure is declared as follows::
struct MessageStruct {
Sig sig;
MessageType type;
MessageClass class;
RingStruct node;
} MessageStruct;
.. c:type:: struct MessageClassStruct *MessageClass
:mps:tag:`class` A message class is an encapsulation of methods. It
encapsulates methods that are applicable to all types of messages
(generic) and methods that are applicable to messages only of a
certain type (type-specific).
:mps:tag:`class.concrete` Concretely a message class is represented by a
:c:type:`MessageClassStruct` (a struct). Clients of the Message module are
expected to allocate storage for and initialise the
:c:type:`MessageClassStruct`. It is expected that such storage will be
allocated and initialised statically.
:mps:tag:`class.one-type` A message class implements exactly one message
type. The identifier for this type is stored in the ``type`` field of
the :c:type:`MessageClassStruct`. Note that the converse is not true: a
single message type may be implemented by two (or more) different
message classes (for example: for two pool classes that require
different implementations for that message type).
:mps:tag:`class.methods.generic` The generic methods are as follows:
* ``delete`` -- used when the message is destroyed (by the client
calling :c:func:`mps_message_discard()`). The class implementation should
finish the message (by calling :c:func:`MessageFinish()`) and storage for
the message should be reclaimed (if applicable).
:mps:tag:`class.methods.specific` The type specific methods are:
:mps:tag:`class.methods.specific.finalization` Specific to
``MessageTypeFinalization``:
* ``finalizationRef`` -- returns a reference to the finalizable object
represented by this message.
:mps:tag:`class.methods.specific.collectionstats` Specific to ``MessageTypeCollectionStats``:
* ``collectionStatsLiveSize`` -- returns the number of bytes (of
objects) that were condemned but survived.
* ``collectionStatsCondemnedSize`` -- returns the number of bytes
condemned in the collection.
* ``collectionStatsNotCondemnedSize`` -- returns the the number of
bytes (of objects) that are subject to a GC policy (that is,
collectable) but were not condemned in the collection.
:mps:tag:`class.sig.double` The :c:type:`MessageClassStruct` has a signature field
at both ends. This is so that if the :c:type:`MessageClassStruct` changes
size (by adding extra methods for example) then any static
initializers will generate errors from the compiler (there will be a
type error causes by initialising a non-signature type field with a
signature) unless the static initializers are changed as well.
:mps:tag:`class.struct` The structure is declared as follows::
typedef struct MessageClassStruct {
Sig sig; /* design.mps.sig */
const char *name; /* Human readable Class name */
/* generic methods */
MessageDeleteMethod delete; /* terminates a message */
/* methods specific to MessageTypeFinalization */
MessageFinalizationRefMethod finalizationRef;
/* methods specific to MessageTypeCollectionStats */
MessageCollectionStatsLiveSizeMethod collectionStatsLiveSize;
MessageCollectionStatsCondemnedSizeMethod collectionStatsCondemnedSize;
MessageCollectionStatsNotCondemnedSizeMethod collectionStatsNotCondemnedSize;
Sig endSig; /* design.mps.message.class.sig.double */
} MessageClassStruct;
:mps:tag:`space.queue` The arena structure is augmented with a structure for
managing for queue of pending messages. This is a ring in the
:c:type:`ArenaStruct`::
struct ArenaStruct
{
...
RingStruct messageRing;
...
}
Functions
.........
.. c:function:: void MessageInit(Arena arena, Message message, MessageClass class)
:mps:tag:`fun.init` Initializes the :c:type:`MessageStruct` pointed to by
``message``. The caller of this function is expected to manage the
store for the :c:type:`MessageStruct`.
.. c:function:: void MessageFinish(Message message)
:mps:tag:`fun.finish` Finishes the :c:type:`MessageStruct` pointed to by
``message``. The caller of this function is expected to manage the
store for the :c:type:`MessageStruct`.
.. c:function:: void MessagePost(Arena arena, Message message)
:mps:tag:`fun.post` Places a message on the queue of an arena.
:mps:tag:`fun.post.precondition` Prior to calling the function, the node
field of the message must be a singleton. After the call to the
function the message will be available for MPS client to access. After
the call to the function the message fields must not be manipulated
except from the message's class's method functions (that is, you
mustn't poke about with the node field in particular).
.. c:function:: void MessageEmpty(Arena arena)
:mps:tag:`fun.empty` Empties the message queue. This function has the same
effect as discarding all the messages on the queue. After calling this
function there will be no messages on the queue.
:mps:tag:`fun.empty.internal-only` This functionality is not exposed to
clients. We might want to expose this functionality to our clients in
the future.
Message life cycle
------------------
:mps:tag:`life` A message will be allocated by a client of the message
module, it will be initialised by calling :c:func:`MessageInit()`. The
client will eventually post the message on the external queue (in fact
most clients will create a message and then immediately post it). The
message module may then apply any of the methods to the message. The
message module will eventually destroy the message by applying the
``delete`` method to it.
Examples
--------
Finalization
............
.. note::
Possibly out of date, see design.mps.finalize and
design.mps.poolmrg instead. David Jones, 1997-08-28.
This subsection is a sketch of how PoolMRG will use Messages for
finalization (see design.mps.poolmrg).
PoolMRG has guardians (see design.mps.poolmrg.guardian). Guardians are
used to manage final references and detect when an object is
finalizable.
The link part of a guardian will include a :c:type:`MessageStruct`.
The :c:type:`MessageStruct` is allocated when the final reference is created
(which is when the referred to object is registered for finalization).
This avoids allocating at the time when the message gets posted (which
might be a tricky, undesirable, or impossible, time to allocate).
PoolMRG has two queues: the entry queue, and the exit queue. The entry
queue will use a ring; the exit queue of MRG will simply be the
external message queue.
The ``delete`` method frees both the link part and the reference part
of the guardian.

403
mps/manual/html/_sources/design/object-debug.txt
View file

@ -1,6 +1,405 @@
.. _design-object-debug:
.. index::
pair: debugging; design
.. _design-object-debug:
.. include:: ../../converted/Object-debug.rst
Debugging features for client objects
=====================================
.. mps:prefix:: design.mps.object-debug
Introduction
------------
:mps:tag:`intro` This is the design for all the various debugging features
that MPS clients (and sometimes MPS developers) can use to discover
what is happening to their objects and the memory space.
:mps:tag:`readership` MPS developers.
Overview
--------
:mps:tag:`over.fenceposts` In its current state, this document mostly talks
about fenceposts, straying a little into tagging where theses features
have an effect on each other.
.. note::
There exist other documents that list other required features, and
propose interfaces and implementations. These will eventually be
folded into this one. Pekka P. Pirinen, 1998-09-10.
Requirements
------------
:mps:tag:`req.fencepost` Try to detect overwrites and underwrites of
allocated blocks by adding fenceposts (source req.product.??? VC++,
req.epcore.fun.debug.support).
:mps:tag:`req.fencepost.size` The fenceposts should be at least 4 bytes on
either side or 8 bytes if on one side only, with an adjustable content
(although VC++ only has 4 bytes with pattern 0xFDFDFDFD, having
unwisely combined the implementation with other debug features).
:mps:tag:`req.fencepost.check` There should be a function to check all the
fenceposts (source req.epcore.fun.debug.support).
:mps:tag:`req.free-block` Try to detect attempts to write and read free
blocks.
:mps:tag:`req.walk` There should be a way to map ("walk") a user function
over all allocated objects (except PS VM objects), possibly only in a
separate debugging variety/mode (source req.epcore.fun.debug.support).
:mps:tag:`req.tag` There should be a way to store at least a word of user
data (a "tag", borrowing the SW term) with every object in debugging
mode, to be used in memory dumps (source req.product.??? VC++).
:mps:tag:`req.tag.walk` The walking function (as required by :mps:ref:`.req.walk`)
should have access to this data (source req.epcore.fun.debug.support).
:mps:tag:`req.dump.aver` It must be possible to perform a memory dump after
an :c:func:`AVER()` has fired. Naturally, if the information required for
the dump has been corrupted, it will fail, as softly as possible
(source @@@@).
.. note::
There are more requirements, especially about memory dumps and
allocation locations. Pekka P. Pirinen, 1998-09-10.
Solution ideas
--------------
:mps:tag:`note.assumptions` I've tried not to assume anything about the
coincidence of manual/automatic, formatted/unformatted, and
ap/mps_alloc. I think those questions deserve to be decided on their
own merits. instead of being constrained by a debug feature.
:mps:tag:`fence.content.repeat` The content of a fencepost could be
specified as a byte/word which used repeatedly to fill the fencepost.
:mps:tag:`fence.content.template` The content could be given as a template
which is of the right size and is simply copied onto the fencepost.
:mps:tag:`fence.walk` :mps:ref:`.req.fencepost.check` requires the ability to find
all the allocated objects. In formatted pools, this is not a problem.
In unformatted pools, we could use the walker. It's a feasible
strategy to bet that any pool that might have to support fenceposting
will also have a walking requirement.
:mps:tag:`fence.tag` Fenceposting also needs to keep track which objects
have fenceposts. unless we manage to do them all. It would be easiest
to put this in the tags.
:mps:tag:`fence.check.object` A function to check the fenceposts on a given
object would be nice.
:mps:tag:`fence.ap` AP's could support fenceposting transparently by having
a mode where :c:func:`mps_reserve()` always goes out-of-line and fills in the
fenceposts (the pool's :c:func:`BufferFill()` method isn't involved). This
would leave the MPS with more freedom of implementation, especially
when combined with some of the other ideas. We think doing a function
call for every allocation is not too bad for debugging.
:mps:tag:`fence.outside-ap` We could also let the client insert their own
fenceposts outside the MPS allocation mechanism. Even if fenceposting
were done like this, we'd still want it to be an MPS feature, so we'd
offer sample C macros for adding the size of the fencepost and filling
in the fencepost pattern. Possibly something like this (while we could
still store the parameters in the pool or allocation point, there
seems little point in doing so in this case, and having them as
explicit parameters to the macros allows the client to specify
constants to gain effiency)::
#define mps_add_fencepost(size, fp_size)
#define mps_fill_fenceposts(obj, size, fp_size, fp_pattern)
The client would need to supply their own fencepost checking function,
obviously, but again we could offer one that matches the sample
macros.
:mps:tag:`fence.tail-only` In automatic pools, the presence of a fencepost
at the head of the allocated block results in the object reference
being an internal pointer. This means that the format or the pool
would need to know about fenceposting and convert between references
and pointers. This would slow down the critical path when fenceposting
is used. This can be ameliorated by putting a fencepost at the tail of
the block only: this obviates the internal pointer problem and could
provide almost the same degree of checking (provided the size was
twice as large), especially in copying pools, where there are normally
no gaps between allocated blocks. In addition to the inescapable
effects on allocation and freeing (including copying and reclaim
thereunder), only scanning would have to know about fenceposts.
:mps:tag:`fence.tail-only.under` Walking over all the objects in the pool
would be necessary to detect underwrites, as one couldn't be sure that
there is a fencepost before any given object (or where it's located
exactly). If the pool were doing the checking, it could be sure: it
would know about alignments and it could put fenceposts in padding
objects (free blocks will have them because they were once allocated)
so there'd be one on either side of any object (except at the head of
a segment, which is not a major problem, and could be fixed by adding
a padding object at the beginning of every segment). This requires
some cleverness to avoid splinters smaller than the fencepost size,
but it can be done.
:mps:tag:`fence.wrapper` On formatted pools, fenceposting could be
implemented by "wrapping" the client-supplied format at creation time.
The wrapper can handle the conversion from the fenceposted object and
back. This will be invisible to the client and gives the added benefit
that the wrapper can validate fenceposts on every format operation,
should it desire. That is, the pool would see the fenceposts as part
of the client object, but the client would only see its object; the
format wrapper would translate between the two. Note that hiding the
fenceposts from scan methods, which are required to take a contiguous
range of objects, is a bit complicated.
:mps:tag:`fence.client-format` The MPS would supply such a wrapper, but
clients could also be allowed to write their own fenceposted formats
(provided they coordinate with allocation, see below). This would make
scanning fenceposted segments more efficient.
:mps:tag:`fence.wrapper.variable` Furthermore, you could create different
classes of fencepost within a pool, because the fencepost itself could
have a variable format. For instance, you might choose to have the
fencepost be minimal (one to two words) for small objects, and more
detailed/complex for large objects (imagining that large objects are
likely vector-ish and subject to overruns). You could get really fancy
and have the fencepost class keyed to the object class (for example,
different allocation points create different classes of fenceposting).
:mps:tag:`fence.wrapper.alloc` Even with a wrapped format, allocation and
freeing would still have know about the fenceposts. If allocation
points are used, either MPS-side (:mps:ref:`.fence.ap`) or client-side
(:mps:ref:`.fence.outside-ap`) fenceposting could be used, with the obvious
modifications.
:mps:tag:`fence.wrapper.alloc.format` We could add three format methods, to
adjust the pointer and the size for alloc and free, to put down the
fenceposts during alloc, and to check them; to avoid slowing down all
allocation, this would require some MOPping to make the format class
affect the choice of the alloc and free methods (see
mail.pekka.1998-06-11.18-18).
:mps:tag:`fence.wrapper.alloc.size` We could just communicate the size of
the fenceposts between the format and the allocation routines, but
then you couldn't use variable fenceposts (.fence.wrapper.variable).
.. note::
All this applies to copying and reclaim in a straight-forward
manner, I think.
:mps:tag:`fence.pool.wrapper` Pools can be wrapped as well. This could be a
natural way to represent/implement the fenceposting changes to the
Alloc and Free methods. [@@@@alignment]
:mps:tag:`fence.pool.new-class` We could simply offer a debugging version of
each pool class (e.g., :c:func:`mps_pool_class_mv_debug()`). As we have seen,
debugging features have synergies which make it advantageous to have a
coordinated implementation, so splitting them up would not just
complicate the client interface, it would also be an implementation
problem; we can turn features on or off with pool init parameters.
:mps:tag:`fence.pool.abstract` We could simply use pool init parameters only
to control all debugging features (optargs would be useful here).
While there migh be subclasses and wrappers internally, the client
would only see a single pool class; in the internal view, this would
be an abstract class, and the parameters would determine which
concrete class actually gets instantiated.
:mps:tag:`tag.out-of-line` It would be nice if tags were stored out-of-line,
so they can be used to study allocation patterns and fragmentation
behaviours. Such an implementation of tagging could also easily be
shared among several pools.
Architecture
------------
:mps:tag:`pool` The implementation is at the pool level, because pools
manage allocated objects. A lot of the code will be generic,
naturally, but the data structures and the control interfaces attach
to pools. In particular, clients will be able to use tagging and
fenceposting separately on each pool.
:mps:tag:`fence.size` Having fenceposts of adjustable size and pattern is
quite useful. We feel that restricting the size to an integral
multiple of the [pool or format?] alignment is harmless and simplifies
the implementation enormously.
:mps:tag:`fence.template` We use templates (:mps:ref:`.fence.content.template`) to
fill in the fenceposts, but we do not give any guarantees about the
location of the fenceposts, only that they're properly aligned. This
leaves us the opportunity to do tail-only fenceposting, if we choose.
:mps:tag:`fence.slop` [see impl.c.dbgpool.FenceAlloc @@@@]
:mps:tag:`fence.check.free` We check the fenceposts when freeing an object.
:mps:tag:`unified-walk` Combine the walking and tagging requirements
(:mps:ref:`.req.tag.walk` and @@@@) into a generic facility for walking and
tagging objects with just one interface and one name: tagging. Also
combine the existing formatted object walker into this metaphor, but
allowing the format and tag parameters of the step function be
optional.
.. note::
This part has not been implemented yet Pekka P. Pirinen,
1998-09-10.
:mps:tag:`init` It simplifies the implementation of both tagging and
fenceposting if they are always on, so that we don't have to keep
track of which objects have been fenceposted and which have not, and
don't have to have three kinds of tags: for user data, for
fenceposting, and for both. So we determine this at pool init time
(and let fenceposting turn on tagging, if necessary).
:mps:tag:`pool-parameters` Fencepost templates and tag formats are passed in
as pool parameters.
:mps:tag:`modularity` While a combined generic implementation of tags and
fenceposts is provided, it is structured so that each part of it could
be implemented by a pool-specific mechanism with a minimum of new
protocol.
.. note::
This will be improved, when we figure out formatted pools -- they
don't need tags for fenceposting.
:mps:tag:`out-of-space` If there's no room for tags, we will not dip into
the reservoir, just fail to allocate the tag. If the alloc call had a
reservoir permit, we let it succeed even without a tag, and just make
sure the free method will not complain if it can't find a tag. If the
call didn't have a reservoir permit, we free the block allocated for
the object and fail the allocation, so that the client gets a chance
to do whatever low-memory actions they might want to do.
.. note::
Should this depend on whether there is anything in the reservoir?
This breaks the one-to-one relationship between tags and objects, so
some checks cannot be made, but we do count the "lost" tags.
.. note::
Need to hash out how to do fenceposting in formatted pools.
Client interface
----------------
:mps:tag:`interface.fenceposting.check`
:c:func:`mps_pool_check_fenceposts()` is a function to check all
fenceposts in a pool (:c:func:`AVER()` if a problem is found)
.. note::
From here on, these are tentative and incomplete.
.. c:function:: mps_res_t mps_fmt_fencepost_wrap(mps_fmt_t *format_return, mps_arena_t arena, mps_fmt_t format, ...)
:mps:tag:`interface.fenceposting.format` A function to wrap a format
(class) to provide fenceposting.
.. c:function:: void (*mps_fmt_adjust_fencepost_t)(size_t *size_io)
:mps:tag:`interface.fenceposting.adjust` A format method to adjust size of a
block about to be allocted to allow for fenceposts.
.. c:function:: void (*mps_fmt_put_fencepost_t)(mps_addr_t * addr_io, size_t size)
:mps:tag:`interface.fenceposting.add` A format method to add a fencepost
around a block about to be allocated. The :c:macro:`NULL` method adds a tail
fencepost.
.. c:function:: mps_bool_t (*mps_fmt_check_fenceposts_t)(mps_addr_t)
:mps:tag:`interface.fenceposting.checker` A format method to check the
fenceposts around an object. The :c:macro:`NULL` method checks tails.
.. c:function:: mps_class_t mps_debug_class(mps_class_t class)
:mps:tag:`interface.fenceposting.pool` A function to wrap a pool class
to provide fenceposting (note absence of arena parameter).
``mps_res_t mps_alloc(mps_addr_t *, mps_pool_t, size_t);``
``mps_res_t mps_alloc_dbg(mps_addr_t *, mps_pool_t, size_t, ...);``
``mps_res_t mps_alloc_dbg_v(mps_addr_t *, mps_pool_t, size_t, va_list);``
:mps:tag:`interface.tags.alloc` Three functions to replace existing
:c:func:`mps_alloc()` (request.???.??? proposes to remove the varargs)
.. c:function:: void (*mps_objects_step_t)(mps_addr_t addr, size_t size, mps_fmt_t format, mps_pool_t pool, void *tag_data, void *p)
.. c:function:: void mps_pool_walk(mps_arena_t arena, mps_pool_t pool, mps_objects_step_t step, void *p)
.. c:function:: void mps_arena_walk(mps_arena_t arena, mps_objects_step_t step, void *p)
:mps:tag:`interface.tags.walker` Functions to walk all the allocated
objects in a pool or an arena (only client pools in this case),
``format`` and ``tag_data`` can be :c:macro:`NULL` (``tag_data`` really wants
to be ``void *``, not :c:type:`mps_addr_t`, because it's stored
together with the internal tag data in an MPS internal pool)
Examples
--------
:mps:tag:`example.debug-alloc` ::
#define MPS_ALLOC_DBG(res_io, addr_io, pool, size)
MPS_BEGIN
static mps_tag_A_s _ts = { __FILE__, __LINE__ };
*res_io = mps_alloc(addr_io, pool, size, _ts_)
MPS_END
Implementation
--------------
:mps:tag:`new-pool` The client interface to control fenceposting
consists of the new classes :c:func:`mps_pool_class_mv_debug()`,
:c:func:`mps_pool_class_epdl_debug()`, and
:c:func:`mps_pool_class_epdr_debug()`, and their new init parameter of
type :c:type:`mps_pool_debug_option_s`.
.. note::
This is a temporary solution, to get it out without writing lots
of new interface. Pekka P. Pirinen, 1998-09-10.
:mps:tag:`new-pool.impl` The debug pools are implemented using the "class
wrapper" :c:func:`EnsureDebugClass()`, which produces a subclass with
modified ``init``, ``finish``, ``alloc``, and ``free`` methods. These
methods are implemented in the generic debug class code
(``impl.c.dbgpool``), and are basically wrappers around the superclass
methods (invoked through the ``pool->class->super`` field). To find
the data stored in the class for the debugging features, they use the
``debugMixin`` method provided by the subclass. So to make a debug
subclass, three things should be provided: a structure definition of
the instance containing a :c:type:`PoolDebugMixinStruct`, a pool class
function that uses :c:func:`EnsureDebugClass()`, and a ``debugMixin`` method
that locates the :c:type:`PoolDebugMixinStruct` within an instance.
:mps:tag:`tags.splay` The tags are stored in a splay tree of tags
allocated from a subsidiary MFS pool. The client needs to specify the
(maximum) size of the client data in a tag, so that the pool can be
created.
.. note::
Lots more should be said, eventually. Pekka P. Pirinen,
1998-09-10.

25
mps/manual/html/_sources/design/old.txt
View file

@ -14,32 +14,55 @@ Old design
.. toctree::
:numbered:
alloc-frame
arena
arenavm
bt
cbs
buffer
check
class-interface
collection
diag
finalize
fix
interface-c
io
lib
lock
locus
message
message-gc
object-debug
pool
poolamc
poolams
poolawl
poollo
poolmfs
poolmrg
poolmvff
prot
protan
protli
protsu
protocol
pthreadext
reservoir
root
scan
seg
shield
splay
sso1al
telemetry
thread-manager
thread-safety
trace
type
version-library
version
vm
vman
vmo1
vmso
writef

61
mps/manual/html/_sources/design/pool.txt
View file

@ -1,6 +1,63 @@
.. _design-pool:
.. index::
pair: pool class mechanism; design
.. _design-pool:
.. include:: ../../converted/pool.rst
Pool and pool class mechanisms
==============================
.. mps:prefix:: design.mps.pool
Definitions
-----------
:mps:tag:`def.outer-structure` The "outer structure" (of a pool) is a C
object of type :c:type:`PoolXXXStruct` or the type ``struct PoolXXXStruct``
itself.
:mps:tag:`def.generic-structure` The "generic structure" is a C object of
type :c:type:`PoolStruct` (found embedded in the outer-structure) or the
type ``struct PoolStruct`` itself.
Defaults
--------
:mps:tag:`align` When initialised, the pool gets the default alignment
(:c:macro:`ARCH_ALIGN`).
:mps:tag:`no` If a pool class doesn't implement a method, and doesn't expect
it to be called, it should use a non-method (``PoolNo*``) which will
cause an assertion failure if they are reached.
:mps:tag:`triv` If a pool class supports a protocol but does not require any
more than a trivial implementation, it should use a trivial method
(``PoolTriv*``) which will do the trivial thing.
:mps:tag:`outer-structure.sig` It is good practice to put the signature for
the outer structure at the end (of the structure). This is because
there's already one at the beginning (in the poolStruct) so putting it
at the end gives some extra fencepost checking.
Requirements
------------
.. note::
Placeholder: must derive the requirements from the architecture.
:mps:tag:`req.fix` :c:func:`PoolFix()` must be fast.
Other
-----
Interface in mpm.h
Types in mpmst.h
See also design.mps.poolclass

801
mps/manual/html/_sources/design/poolamc.txt
View file

@ -1,7 +1,798 @@
.. index::
pair: AMC pool; design
single: pool; AMC design
.. _design-poolamc:
.. include:: ../../converted/poolamc.rst
.. index::
pair: AMC pool class; design
single: pool class; AMC design
AMC pool class
==============
.. mps:prefix:: design.mps.poolamc
pair: AMC pool class; design
single: pool class; AMC design
Introduction
~~~~~~~~~~~~
:mps:tag:`intro` This document contains a guide (:mps:ref:`.guide`) to the MPS AMC
pool class, followed by the historical initial design
(:mps:ref:`.initial-design`).
:mps:tag:`readership` Any MPS developer.
Guide
~~~~~
:mps:tag:`guide` The AMC pool class is a general-purpose automatic
(collecting) pool class. It is intended for most client objects. AMC
is "Automatic, Mostly Copying": it preserves objects by copying,
except when an ambiguous reference 'nails' the object in place. It is
generational. Chain: specify capacity and mortality of generations 0
to *N* − 1. Survivors from generation *N* − 1 get promoted into an
arena-wide "top" generation (often anachronistically called the
"dynamic" generation, which was the term on the Lisp Machine).
Segment states
--------------
:mps:tag:`seg.state` AMC segments are in one of three states: "mobile",
"boarded", or "stuck".
:mps:tag:`seg.state.mobile` Segments are normally **mobile**: all objects on
the seg are un-nailed, and thus may be preserved by copying.
:mps:tag:`seg.state.boarded` An ambiguous reference to any address within an
segment makes that segment **boarded**: a nailboard is allocated to
record ambiguous references ("nails"), but un-nailed objects on the
segment are still preserved by copying.
:mps:tag:`seg.state.stuck` Stuck segments only occur in emergency tracing: a
discovery fix to an object in a mobile segment is recorded in the only
non-allocating way available: by making the entire segment **stuck**.
Pads
----
(See job001809_ and job001811_, and mps/branch/2009-03-31/padding.)
.. _job001809: http://www.ravenbrook.com/project/mps/issue/job001809/
.. _job001811: http://www.ravenbrook.com/project/mps/issue/job001811/
:mps:tag:`pad` A pad is logically a trivial client object. Pads are created
by the MPS asking the client's format code to create them, to fill up
a space in a segment. Thereafter, the pad appears to the MPS as a
normal client object (that is: the MPS cannot distinguish a pad from a
client object).
:mps:tag:`pad.reason` AMC creates pads for three reasons: buffer empty
fragment (BEF), large segment padding (LSP), and non-mobile reclaim
(NMR). (Large segment pads were new with job001811_.)
:mps:tag:`pad.reason.bef` Buffer empty fragment (BEF) pads are made by
:c:func:`AMCBufferEmpty()` whenever it detaches a non-empty buffer from an
AMC segment. Buffer detachment is most often caused because the buffer
is too small for the current buffer reserve request (which may be
either a client requested or a forwarding allocation). Detachment may
happen for other reasons, such as trace flip.
:mps:tag:`pad.reason.lsp` Large segment padding (LSP) pads are made by
:c:func:`AMCBufferFill()` when the requested fill size is "large" (see `The
LSP payoff calculation`_ below). :c:func:`AMCBufferFill()` fills the buffer
to exactly the size requested by the current buffer reserve operation;
that is: it does not round up to the whole segment size. This prevents
subsequent small objects being placed in the same segment as a single
very large object. If the buffer fill size is less than the segment
size, :c:func:`AMCBufferFill()` fills any remainder with an large segment
pad.
:mps:tag:`pad.reason.nmr` Non-mobile reclaim (NMR) pads are made by
:c:func:`amcReclaimNailed()`, when performing reclaim on a non-mobile (that
is, either boarded or stuck) segment:
The more common NMR scenario is reclaim of a boarded segment after a
non-emergency trace. Ambiguous references into the segment are
recorded as nails. Subsequent exact references to a nailed object do
nothing further, but exact refs that do not match a nail cause
preserve-by-copy and leave a forwarding object. Unreachable objects
are not touched during the scan+fix part of the trace. On reclaim,
only nailed objects need to be preserved; others (namely forwarding
pointers and unreachable objects) are replaced by an NMR pad. (Note
that a BEF or LSP pad appears to be an unreachable object, and is
therefore overwritten by an NMR pad).
The less common NMR scenario is after emergency tracing. Boarded
segments still occur; they may have nailed objects from ambiguous
references, forwarding objects from pre-emergency exact fixes, nailed
objects from mid-emergency exact fixes, and unpreserved objects;
reclaim is as in the non-emergency case. Stuck segments may have
forwarding objects from pre-emergency exact fixes, objects from
mid-emergency fixes, and unreachable objects -- but the latter two are
not distinguishable because there is no nailboard. On reclaim, all
objects except forwarding pointers are preserved; each forwarding
object is replaced by an NMR pad.
If :c:func:`amcReclaimNailed()` finds no objects to be preserved then it
calls :c:func:`SegFree()` (new with job001809_).
Placement pads are okay
-----------------------
Placement pads are the BEF and LSP pads created in "to-space" when
placing objects into segments. This wasted space is an expected
space-cost of AMC's naive (but time-efficient) approach to placement
of objects into segments. This is normally not a severe problem. (The
worst case is a client that always requests ``ArenaAlign() + 1`` byte
objects: this has a nearly 100% overhead).
Retained pads could be a problem
--------------------------------
Retained pads are the NMR pads stuck in "from-space": non-mobile
segments that were condemned but have preserved-in-place objects
cannot be freed by :c:func:`amcReclaimNailed()`. The space around the
preserved objects is filled with NMR pads.
In the worst case, retained pads could waste an enormous amount of
space! A small (one-byte) object could retain a multi-page segment for
as long as the ambiguous reference persists; that is: indefinitely.
Imagine a 256-page (1 MiB) segment containing a very large object
followed by a handful of small objects. An ambiguous reference to one
of the small objects will unfortunately cause the entire 256-page
segment to be retained, mostly as an NMR pad; this is a massive
overhead of wasted space.
AMC mitigates this worst-case behaviour, by treating large segments
specially.
Small, medium, and large segments
---------------------------------
AMC categorises segments as **small** (one page), **medium**
(several pages), or **large** (``AMCLargeSegPAGES`` or more)::
pages = SegSize(seg) / ArenaAlign(arena);
if(pages == 1) {
/* small */
} else if(pages < AMCLargeSegPAGES) {
/* medium */
} else {
/* large */
}</code></pre></blockquote>
``AMCLargeSegPAGES`` is currently 8 -- see `The LSP payoff
calculation`_ below.
AMC might treat "Large" segments specially, in two ways:
- :mps:tag:`large.single-reserve` A large segment is only used for a single
(large) buffer reserve request; the remainder of the segment (if
any) is immediately padded with an LSP pad.
- :mps:tag:`large.lsp-no-retain` Nails to such an LSP pad do not cause
AMCReclaimNailed() to retain the segment.
:mps:ref:`.large.single-reserve` is implemented. See job001811_.
:mps:ref:`.large.lsp-no-retain` is **not** currently implemented.
The point of :mps:ref:`.large.lsp-no-retain` would be to avoid retention of
the (large) segment when there is a spurious ambiguous reference to
the LSP pad at the end of the segment. Such an ambiguous reference
might happen naturally and repeatably if the preceding large object is
an array, the array is accessed by an ambiguous element pointer (for
example, on the stack), and the element pointer ends up pointing just
off the end of the large object (as is normal for sequential element
access in C) and remains with that value for a while. (Such an
ambiguous reference could also occur by chance, for example, by
coincidence with an ``int`` or ``float``, or when the stack grows to
include old unerased values).
Implementing :mps:ref:`.large.lsp-no-retain` is a little tricky. A pad is
indistinguishable from a client object, so AMC has no direct way to
detect, and safely ignore, the final LSP object in the seg. If AMC
could *guarantee* that the single buffer reserve
(:mps:ref:`.large.single-reserve`) is only used for a single *object*, then
:c:func:`AMCReclaimNailed()` could honour a nail at the start of a large seg
and ignore all others; this would be extremely simple to implement.
But AMC cannot guarantee this, because in the MPS Allocation Point
Protocol the client is permitted to make a large buffer reserve and
then fill it with many small objects. In such a case, AMC must honour
all nails (if the buffer reserve request was an exact multiple of
:c:func:`ArenaAlign()`), or all nails except to the last object (if there
was a remainder filled with an LSP pad). Because an LSP pad cannot be
distinguished from a client object, and the requested allocation size
is not recorded, AMC cannot distinguish these two conditions at
reclaim time. Therefore AMC must record whether or not the last object
in the seg is a pad, in order to ignore nails to it. This could be
done by adding a flag to :c:type:`AMCSegStruct`. (This can be done without
increasing the structure size, by making the ``Bool new`` field
smaller than its current 32 bits.)
The LSP payoff calculation
--------------------------
The LSP fix for job001811_ treats large segments differently. Without
it, after allocating a very large object (in a new very large
multi-page segment), MPS would happily place subsequent small objects
in any remaining space at the end of the segment. This would risk
pathological fragmentation: if these small objects were systematically
preserved by ambiguous refs, enormous NMR pads would be retained along
with them.
The payoff calculation is a bit like deciding whether or not to
purchase insurance. For single-page and medium-sized segments, we go
ahead and use the remaining space for subsequent small objects. This
is equivalent to choosing **not** to purchase insurance. If the small
objects were to be preserved by ambiguous refs, the retained NMR pads
would be big, but not massive. We expect such ambiguous refs to be
uncommon, so we choose to live with this slight risk of bad
fragmentation. The benefit is that the remaining space is used.
For large segments, we decide that the risk of using the remainder is
just too great, and the benefit too small, so we throw it away as an
LSP pad. This is equivalent to purchasing insurance: we choose to pay
a known small cost every time, to avoid risking an occasional
disaster.
To decide what size of segment counts as "large", we must decide how
much uninsured risk we can tolerate, versus how much insurance cost we
can tolerate. The likelihood of ambiguous references retaining objects
is entirely dependent on client behaviour. However, as a sufficient
"one size fits all" policy, I (RHSK 2009-09-14) have judged that
segments smaller than eight pages long do not need to be treated as
large: the insurance cost to "play safe" would be considerable
(wasting up to one page of remainder per seven pages of allocation),
and the fragmentation overhead risk is not that great (at most eight
times worse than the unavoidable minimum). So ``AMCLargeSegPAGES`` is
defined as 8 in config.h. As long as the assumption that most segments
are not ambiguously referenced remains correct, I expect this policy
will be satisfactory.
To verify that this threshold is acceptable for a given client,
poolamc.c calculates metrics; see `Feedback about retained pages`_
below. If this one-size-fits-all approach is not satisfactory,
``AMCLargeSegPAGES`` could be made a client-tunable parameter.
Retained pages
--------------
The reasons why a segment and its pages might be retained are:
1. ambiguous reference to first-obj: unavoidable page retention (only
the mutator can reduce this, if they so wish, by nulling out ambig
referencess);
2. ambiguous reference to rest-obj: tuning MPS LSP policy could
mitigate this, reducing the likelihood of rest-objs being
co-located with large first-objs;
3. ambiguous reference to final pad: implementing
:mps:ref:`.large.lsp-no-retain` could mitigate this;
4. ambiguous reference to other (NMR) pad: hard to mitigate, as pads
are indistinguishable from client objects;
5. emergency trace;
6. non-object-aligned ambiguous ref: fixed by job001809_;
7. other reason (for example, buffered at flip): not expected to be a
problem.
This list puts the reasons that are more "obvious" to the client
programmer first, and the more obscure reasons last.
Feedback about retained pages
-----------------------------
(New with job001811_). AMC now accumulates counts of pages condemned
and retained during a trace, in categories according to size and
reason for retention, and emits diagnostic at trace-end via the
``pool->class->traceEnd`` method. See comments on the
:c:type:`PageRetStruct` in poolamc.c. These page-based metrics are not as
precise as actually counting the size of objects, but they require
much less intrusive code to implement, and should be sufficient to
assess whether AMC's page retention policies and behaviour are
acceptable.
Initial design
~~~~~~~~~~~~~~
Introduction
------------
:mps:tag:`intro` This is the design of the AMC Pool Class. AMC stands for
Automatic Mostly-Copying. This design is highly fragmentory and some
may even be sufficiently old to be misleading.
:mps:tag:`readership` The intended readership is any MPS developer.
Overview
--------
:mps:tag:`overview` This class is intended to be the main pool class used by
Harlequin Dylan. It provides garbage collection of objects (hence
"automatic"). It uses generational copying algorithms, but with some
facility for handling small numbers of ambiguous references. Ambiguous
references prevent the pool from copying objects (hence "mostly
copying"). It provides incremental collection.
.. note::
A lot of this design is awesomely old. David Jones, 1998-02-04.
Definitions
-----------
:mps:tag:`def.grain` Grain. An quantity of memory which is both aligned to
the pool's alignment and equal to the pool's alignment in size. That
is, the smallest amount of memory worth talking about.
Segments
--------
:mps:tag:`seg.class` AMC allocates segments of class :c:type:`AMCSegClass`, which
is a subclass of :c:type:`GCSegClass`. Instances contain a ``segTypeP``
field, which is of type ``int*``.
:mps:tag:`seg.gen` AMC organizes the segments it manages into generations.
:mps:tag:`seg.gen.map` Every segment is in exactly one generation.
:mps:tag:`seg.gen.ind` The segment's ``segTypeP`` field indicates which
generation (that the segment is in) (an :c:type:`AMCGenStruct` see blah
below).
:mps:tag:`seg.typep` The ``segTypeP`` field actually points to either the
type field of a generation or to the type field of a nail board.
:mps:tag:`seg.typep.distinguish` The ``type`` field (which can be accessed
in either case) determines whether the ``segTypeP`` field is pointing
to a generation or to a nail board.
:mps:tag:`seg.gen.get` The map from segment to generation is implemented by
:c:func:`AMCSegGen()` which deals with all this.
Fixing and nailing
------------------
.. note::
This section contains placeholders for design rather than design
really. David Jones, 1998-02-04.
:mps:tag:`nailboard` AMC uses a nail board structure for recording ambiguous
references to segments. A nail board is a bit table with one bit per
grain in the segment.
:mps:tag:`nailboard.create` Nail boards are allocated dynamically whenever a
segment becomes newly ambiguously referenced.
:mps:tag:`nailboard.destroy` They are deallocated during reclaim. Ambiguous
fixes simply set the appropriate bit in this table. This table is used
by subsequent scans and reclaims in order to work out what objects
were marked.
:mps:tag:`nailboard.emergency` During emergency tracing two things relating
to nail boards happen that don't normally:
1. :mps:tag:`nailboard.emergency.nonew` Nail boards aren't allocated when we
have new ambiguous references to segments.
:mps:tag:`nailboard.emergency.nonew.justify` We could try and allocate a
nail board, but we're in emergency mode so short of memory so it's
unlikely to succeed, and there would be additional code for yet
another error path which complicates things.
2. :mps:tag:`nailboard.emergency.exact` nail boards are used to record exact
references in order to avoid copying the objects.
:mps:tag:`nailboard.hyper-conservative` Not creating new nail boards
(:mps:ref:`.nailboard.emergency.nonew` above) means that when we have a new
reference to a segment during emergency tracing then we nail the
entire segment and preserve everything in place.
:mps:tag:`fix.nail.states` Partition the segment states into four sets:
1. white segment and not nailed (and has no nail board);
2. white segment and nailed and has no nail board;
3. white segment and nailed and has nail board;
4. the rest.
:mps:tag:`fix.nail.why` A segment is recorded as being nailed when either
there is an ambiguous reference to it, or there is an exact reference
to it and the object couldn't be copied off the segment (because there
wasn't enough memory to allocate the copy). In either of these cases
reclaim cannot simply destroy the segment (usually the segment will
not be destroyed because it will have live objects on it, though see
:mps:ref:`.nailboard.limitations.middle` below). If the segment is nailed then
we might be using a nail board to mark objects on the segment.
However, we cannot guarantee that being nailed implies a nail board,
because we might not be able to allocate the nail board. Hence all
these states actually occur in practice.
:mps:tag:`fix.nail.distinguish` The nailed bits in the segment descriptor
(:c:type:`SegStruct`) are used to record whether a segment is nailed or not.
The ``segTypeP`` field of the segment either points to (the "type"
field of) an ``AMCGen`` or to an ``AMCNailBoard``, the type field can
be used to determine which of these is the case. (see :mps:ref:`.seg.typep`
above).
:mps:tag:`nailboard.limitations.single` Just having a single nail board per
segment prevents traces from improving on the findings of each other:
a later trace could find that a nailed object is no longer nailed or
even dead. Until the nail board is discarded, that is.
:mps:tag:`nailboard.limitations.middle` An ambiguous reference into the
middle of an object will cause the segment to survive, even if there
are no surviving objects on it.
:mps:tag:`nailboard.limitations.reclaim` :c:func:`AMCReclaimNailed()` could cover
each block of reclaimed objects between two nailed objects with a
single padding object, speeding up further scans.
Emergency tracing
-----------------
:mps:tag:`emergency.fix` :c:func:`AMCFixEmergency()` is at the core of AMC's
emergency tracing policy (unsurprisingly). :c:func:`AMCFixEmergency()`
chooses exactly one of three options:
1. use the existing nail board structure to record the fix;
2. preserve and nail the segment in its entirety;
3. snapout an exact (or high rank) pointer to a broken heart to the
broken heart's forwarding pointer.
If the rank of the reference is ``RankAMBIG`` then it either does (1)
or (2) depending on wether there is an existing nail board or not.
Otherwise (the rank is exact or higher) if there is a broken heart it
is used to snapout the pointer. Otherwise it is as for an
``RankAMBIG`` reference: we either do (1) or (2).
:mps:tag:`emergency.scan` This is basically as before, the only complication
is that when scanning a nailed segment we may need to do multiple
passes, as :c:func:`FixEmergency()` may introduce new marks into the nail
board.
Buffers
-------
:mps:tag:`buffer.class` AMC uses buffer of class :c:type:`AMCBufClass` (a subclass
of SegBufClass).
:mps:tag:`buffer.gen` Each buffer allocates into exactly one generation.
:mps:tag:`buffer.field.gen` ``AMCBuf`` buffer contain a gen field which
points to the generation that the buffer allocates into.
:mps:tag:`buffer.fill.gen` :c:func:`AMCBufferFill()` uses the generation (obtained
from the ``gen`` field) to initialise the segment's ``segTypeP`` field
which is how segments get allocated in that generation.
:mps:tag:`buffer.condemn` We condemn buffered segments, but not the contents
of the buffers themselves, because we can't reclaim uncommitted
buffers (see design.mps.buffer for details). If the segment has a
forwarding buffer on it, we detach it.
.. note::
Why? Forwarding buffers are detached because they used to cause
objects on the same segment to not get condemned, hence caused
retention of garbage. Now that we condemn the non-buffered portion
of buffered segments this is probably unnecessary. David Jones,
1998-06-01.
But it's probably more efficient than keeping the buffer on the
segment, because then the other stuff gets nailed -- Pekka P.
Pirinen, 1998-07-10.
If the segment has a mutator buffer on it, we nail the buffer. If the
buffer cannot be nailed, we give up condemning, since nailing the
whole segment would make it survive anyway. The scan methods skip over
buffers and fix methods don't do anything to things that have already
been nailed, so the buffer is effectively black.
Types
-----
:mps:tag:`struct` :c:type:`AMCStruct` is the pool class AMC instance structure.
:mps:tag:`struct.pool` Like other pool class instances, it contains a
:c:type:`PoolStruct` containing the generic pool fields.
:mps:tag:`struct.format` The ``format`` field points to a :c:type:`Format`
structure describing the object format of objects allocated in the
pool. The field is intialized by :c:func:`AMCInit()` from a parameter, and
thereafter it is not changed until the pool is destroyed.
.. note::
Actually the format field is in the generic :c:type:`PoolStruct` these
days. David Jones, 1998-09-21.
.. note::
There are lots more fields here.
Generations
-----------
:mps:tag:`gen` Generations partition the segments that a pool manages (see
:mps:ref:`.seg.gen.map` above).
:mps:tag:`gen.collect` Generations are more or less the units of
condemnation in AMC. And also the granularity for forwarding (when
copying objects during a collection): all the objects which are copied
out of a generation use the same forwarding buffer for allocating the
new copies, and a forwarding buffer results in allocation in exactly
one generation.
:mps:tag:`gen.rep` Generations are represented using an :c:type:`AMCGenStruct`
structure.
:mps:tag:`gen.create` All the generation are create when the pool is created
(during :c:func:`AMCInitComm()`).
:mps:tag:`gen.manage.ring` An AMC's generations are kept on a ring attached
to the :c:type:`AMCStruct` (the ``genRing`` field).
:mps:tag:`gen.manage.array` They are also kept in an array which is
allocated when the pool is created and attached to the :c:type:`AMCStruct`
(the gens field holds the number of generations, the ``gen`` field
points to an array of ``AMCGen``).
.. note::
it seems to me that we could probably get rid of the ring. David
Jones, 1998-09-22.
:mps:tag:`gen.number` There are ``AMCTopGen + 2`` generations in total.
"normal" generations numbered from 0 to ``AMCTopGen`` inclusive and an
extra "ramp" generation (see :mps:ref:`.gen.ramp` below).
:mps:tag:`gen.forward` Each generation has an associated forwarding buffer
(stored in the ``forward`` field of ``AMCGen``). This is the buffer
that is used to forward objects out of this generation. When a
generation is created in :c:func:`AMCGenCreate()`, its forwarding buffer has
a null ``p`` field, indicating that the forwarding buffer has no
generation to allocate in. The collector will assert out (in
:c:func:`AMCBufferFill()` where it checks that ``buffer->p`` is an
``AMCGen``) if you try to forward an object out of such a generation.
:mps:tag:`gen.forward.setup` All the generation's forwarding buffer's are
associated with generations when the pool is created (just after the
generations are created in :c:func:`AMCInitComm()`).
Ramps
-----
:mps:tag:`ramp` Ramps usefully implement the begin/end
:c:func:`mps_alloc_pattern_ramp()` interface.
:mps:tag:`gen.ramp` To implement ramping (request.dylan.170423), AMC uses a
special "ramping mode", where promotions are redirected. One
generation is designated the "ramp generation" (``amc->rampGen`` in
the code).
:mps:tag:`gen.ramp.ordinary` Ordinarily, that is whilst not ramping, objects
are promoted into the ramp generation from younger generations and are
promoted out to older generations. The generation that the ramp
generation ordinarily promotes into is designated the "after-ramp
generation" (``amc->afterRampGen``).
:mps:tag:`gen.ramp.particular` the ramp generation is the second oldest
generation and the after-ramp generation is the oldest generation.
:mps:tag:`gen.ramp.possible` In alternative designs it might be possible to
make the ramp generation a special generation that is only promoted
into during ramping, however, this is not done.
:mps:tag:`gen.ramp.ramping` The ramp generation is promoted into itself
during ramping mode;
:mps:tag:`gen.ramp.after` after this mode ends, the ramp generation is
promoted into the after-ramp generation as usual.
:mps:tag:`gen.ramp.after.once` Care is taken to
ensure that there is at least one collection where stuff is promoted
from the ramp generation to the after-ramp generation even if ramping
mode is immediately re-entered.
:mps:tag:`ramp.mode` This behaviour is controlled in a slightly convoluted
manner by a state machine. The rampMode field of the pool forms an
important part of the state of the machine.
There are five states: OUTSIDE, BEGIN, RAMPING, FINISH, and
COLLECTING. These appear in the code as ``RampOUTSIDE`` and so on.
:mps:tag:`ramp.state.cycle.usual` The usual progression of states is a
cycle: OUTSIDE → BEGIN → RAMPING → FINISH → COLLECTING → OUTSIDE.
:mps:tag:`ramp.count` The pool just counts the number of APs that have begun
ramp mode (and not ended). No state changes occur unless this count
goes from 0 to 1 (starting the first ramp) or from 1 to 0 (leaving the
last ramp). In other words, all nested ramps are ignored (see code in
:c:func:`AMCRampBegin()` and :c:func:`AMCRampEnd()`).
:mps:tag:`ramp.state.invariant.count` In the OUTSIDE state the count must be
zero. In the BEGIN and RAMPING states the count must be greater than
zero. In the FINISH and COLLECTING states the count is not
constrained.
:mps:tag:`ramp.state.invariant.forward` When in OUTSIDE, BEGIN, or
COLLECTING, the ramp generation forwards to the after-ramp generation.
When in RAMPING or FINISH, the ramp generation forwards to itself.
:mps:tag:`ramp.outside` The pool is initially in the OUTSIDE state. The only
transition away from the OUTSIDE state is to the BEGIN state, when a
ramp is entered.
:mps:tag:`ramp.begin` When the count goes up from zero, the state moves from
COLLECTING or OUTSIDE to BEGIN.
:mps:tag:`ramp.begin.leave` We can leave the BEGIN state to either the
OUTSIDE or the RAMPING state.
:mps:tag:`ramp.begin.leave.outside` We go to OUTSIDE if the count drops to 0
before a collection starts. This shortcuts the usual cycle of states
for small enough ramps.
:mps:tag:`ramp.begin.leave.ramping` We enter the RAMPING state if a
collection starts that condemns the ramp generation (pedantically when
a new GC begins, and a segment in the ramp generation is condemned, we
leave the BEGIN state, see AMCWhiten). At this point we switch the
ramp generation to forward to itself (:mps:ref:`.gen.ramp.ramping`).
:mps:tag:`ramp.ramping.leave` We leave the RAMPING state and go to the
FINISH state when the ramp count goes back to zero. Thus, the FINISH
state indicates that we have started collecting the ramp generation
while inside a ramp which we have subsequently finished.
:mps:tag:`ramp.finish.remain` We remain in the FINISH state until we next
start to collect the ramp generation (condemn it), regardless of
entering or leaving any ramps. This ensures that the ramp generation
will be collected to the after-ramp generation at least once.
:mps:tag:`ramp.finish.leave` When we next condemn the ramp genearation, we
move to the COLLECTING state. At this point the forwarding generations
are switched back so that the ramp generation promotes into the
after-ramp generation on this collection.
:mps:tag:`ramp.collecting.leave` We leave the COLLECTING state when the GC
enters reclaim (specifically, when a segment in the ramp generation is
reclaimed), or when we begin another ramp. Ordinarily we enter the
OUTSIDE state, but if the client has started a ramp then we go
directly to the BEGIN state.
_`.ramp.collect-all` There used to be two flavours of ramps: the
normal one and the collect-all flavour that triggered a full GC after
the ramp end. This was a hack for producing certain Dylan statistics,
and no longer has any effect (the flag is passed to
:c:func:`AMCRampBegin()`, but ignored there).
Headers
-------
:mps:tag:`header` AMC supports a fixed-size header on objects, with the
client pointers pointing after the header, rather than the base of the
memory block. See format documentation for details of the interface.
:mps:tag:`header.client` The code mostly deals in client pointers, only
computing the base and limit of a block when these are needed (such as
when an object is copied). In several places, the code gets a block of
some sort, a segment or a buffer, and creates a client pointer by
adding the header length (``pool->format->headerLength``).
:mps:tag:`header.fix` There are two versions of the fix method, due to its
criticality, with (:c:func:`AMCHeaderFix()`) and without (:c:func:`AMCFix()`)
headers. The correct one is selected in :c:func:`AMCInitComm()`, and placed
in the pool's fix field. This is the main reason why fix methods
dispatch through the instance, rather than the class like all other
methods.
Old and aging notes below here
------------------------------
.. c:function:: void AMCFinish(Pool pool)
:mps:tag:`finish.forward` If the pool is being destroyed it is OK to destroy
the forwarding buffers, as the condemned set is about to disappear.
.. c:function:: void AMCBufferEmpty(Pool pool, Buffer buffer, Addr init, Addr limit)
:mps:tag:`flush` Removes the connexion between a buffer and a group, so that
the group is no longer buffered, and the buffer is reset and will
cause a refill when next used.
:mps:tag:`flush.pad` The group is padded out with a dummy object so that it
appears full.
:mps:tag:`flush.expose` The buffer needs exposing before writing the padding
object onto it. If the buffer is being used for forwarding it might
already be exposed, in this case the segment attached to it must be
covered when it leaves the buffer. See :mps:ref:`.fill.expose`.
:mps:tag:`flush.cover` The buffer needs covering whether it was being used
for forwarding or not. See :mps:ref:`.flush.expose`.
.. c:function:: Res AMCBufferFill(Addr *baseReturn, Addr *limitReturn, Pool pool, Buffer buffer, Size size, Bool withReservoirPermit)
:mps:tag:`fill` Reserve was called on an allocation buffer which was reset,
or there wasn't enough room left in the buffer. Allocate a group for
the new object and attach it to the buffer.
:mps:tag:`fill.expose` If the buffer is being used for forwarding it may be
exposed, in which case the group attached to it should be exposed. See
:mps:ref:`.flush.cover`.
.. c:function:: Res AMCFix(Pool pool, ScanState ss, Seg seg, Ref *refIO)
:mps:tag:`fix` Fix a reference to the pool.
Ambiguous references lock down an entire segment by removing it
from old-space and also marking it grey for future scanning.
Exact, final, and weak references are merged because the action for an
already forwarded object is the same in each case. After that
situation is checked for, the code diverges.
Weak references are either snapped out or replaced with
``ss->weakSplat`` as appropriate.
Exact and final references cause the referenced object to be copied to
new-space and the old copy to be forwarded (broken-heart installed) so
that future references are fixed up to point at the new copy.
:mps:tag:`fix.exact.expose` In order to allocate the new copy the forwarding
buffer must be exposed. This might be done more efficiently outside
the entire scan, since it's likely to happen a lot.
:mps:tag:`fix.exact.grey` The new copy must be at least as grey as the old
as it may have been grey for some other collection.
.. c:function:: Res AMCScan(Bool *totalReturn, ScanState ss, Pool pool, Seg seg)
:mps:tag:`scan` Searches for a group which is grey for the trace and scans
it. If there aren't any, it sets the finished flag to true.
.. c:function:: void AMCReclaim(Pool pool, Trace trace, Seg seg)
:mps:tag:`reclaim` After a trace, destroy any groups which are still
condemned for the trace, because they must be dead.
:mps:tag:`reclaim.grey` Note that this might delete things which are grey
for other collections. This is OK, because we have conclusively proved
that they are dead -- the other collection must have assumed they were
alive. There might be a problem with the accounting of grey groups,
however.
:mps:tag:`reclaim.buf` If a condemned group still has a buffer attached, we
can't destroy it, even though we know that there are no live objects
there. Even the object the mutator is allocating is dead, because the
buffer is tripped.

465
mps/manual/html/_sources/design/poolams.txt Normal file
View file

@ -0,0 +1,465 @@
.. _design-poolams:
.. index::
pair: AMS pool class; design
single: pool class; AMS design
AMS pool class
==============
.. mps:prefix:: design.mps.poolams
pair: AMS pool class; design
single: pool class; AMS design
Introduction
------------
:mps:tag:`intro` This is the design of the AMS pool class.
:mps:tag:`readership` MM developers.
:mps:tag:`source` design.mps.buffer, design.mps.trace, design.mps.scan,
design.mps.action and design.mps.class-interface [none of these were
actually used -- pekka 1998-04-21]. No requirements doc [we need a
req.mps that captures the commonalities between the products -- pekka
1998-01-27].
Overview
--------
:mps:tag:`overview` This document describes the design of the AMS (Automatic
Mark-and-Sweep) pool class. The AMS pool is a proof-of-concept design
for a mark-sweep pool in the MPS. It's not meant to be efficient, but
it could serve as a model for an implementation of a more advanced
pool (such as EPVM).
Requirements
------------
:mps:tag:`req.mark-sweep` The pool must use a mark-and-sweep GC algorithm.
:mps:tag:`req.colour` The colour representation should be as efficient as
possible.
:mps:tag:`req.incremental` The pool must support incremental GC.
:mps:tag:`req.ambiguous` The pool must support ambiguous references to
objects in it (but ambiguous references into the middle of an object
do not preserve the object).
:mps:tag:`req.format` The pool must be formatted, for generality.
:mps:tag:`req.correct` The design and the implementation should be simple
enough to be seen to be correct.
:mps:tag:`req.simple` Features not related to mark-and-sweep GC should
initially be implemented as simply as possible, in order to save
development effort.
:mps:tag:`not-req.grey` We haven't figured out how buffers ought to work
with a grey mutator, so we use :mps:ref:`.req.correct` to allow us to design a
pool that doesn't work in that phase. This is acceptable as long as we
haven't actually implemented grey mutator collection.
Architecture
------------
Subclassing
...........
:mps:tag:`subclass` Since we expect to have many mark-and-sweep pools, we
build in some protocol for subclasses to modify various aspects of the
behaviour. Notably there's a subclassable segment class, and a
protocol for performing iteration.
Allocation
..........
:mps:tag:`align` We divide the segments in grains, each the size of the
format alignment. :mps:tag:`alloc-bit-table` We keep track of allocated
grains using a bit table. This allows a simple implementation of
allocation and freeing using the bit table operators, satisfying
:mps:ref:`.req.simple`, and can simplify the GC routines. Eventually, this
should use some sophisticated allocation technique suitable for
non-moving automatic pools.
:mps:tag:`buffer` We use buffered allocation, satisfying
:mps:ref:`.req.incremental`. The AMC buffer technique is reused, although it
is not suitable for non-moving pools, but req.simple allows us to do
that for now.
:mps:tag:`extend` If there's no space in any existing segment, a new segment
is allocated. The actual class is allowed to decide the size of the
new segment.
:mps:tag:`no-alloc` Do not support :c:func:`PoolAlloc()`, because we can't support
one-phase allocation for a scannable pool (unless we disallow
incremental collection). For exact details, see design.mps.buffer.
:mps:tag:`no-free` Do not support :c:func:`PoolFree()`, because automatic pools
don't need explicit free and having it encourages clients to use it
(and therefore to have dangling pointers, double frees, and other
memory management errors.)
Colours
.......
:mps:tag:`colour` Objects in a segment which is *not* condemned (for some
trace) take their colour (for this trace) from the segment.
:mps:tag:`colour.object` Since we need to implement a non-copying GC, we
keep track of the colour of each object in a condemned segment
separately. For this, we use bit tables with a bit for each grain.
This format is fast to access, has better locality than mark bits in
the objects themselves, and allows cheap interoperation with the
allocation bit table.
:mps:tag:`colour.encoding` As to the details, we follow
analysis.non-moving-colour(3), implementing both the alloc-white
sharing option described in
analysis.non-moving-colour.constraint.reclaim.white-free-bit and the
vanilla three-table option, because the former cannot work with
interior pointers. However, the colour encoding in both is the same.
:mps:tag:`ambiguous.middle` We will allow ambiguous references into the
middle of an object (as required by :mps:ref:`.req.ambiguous`), using the
trick in analysis.non-moving-colour.interior.ambiguous-only to speed
up scanning.
:mps:tag:`interior-pointer` Note that non-ambiguous interior pointers are
outlawed.
:mps:tag:`colour.alloc` Objects are allocated black. This is the most
efficient alternative for traces in the black mutator phase, and
.not-req.grey means that's sufficient.
.. note::
Some day, we need to think about allocating grey or white during
the grey mutator phase.
Scanning
........
:mps:tag:`scan.segment` The tracer protocol requires (for segment barrier
hits) that there is a method for scanning a segment and turning all
grey objects on it black. This cannot be achieved with a single
sequential sweep over the segment, since objects that the sweep has
already passed may become grey as later objects are scanned.
:mps:tag:`scan.graph` For a non-moving GC, it is more efficient to trace
along the reference graph than segment by segment. It also allows
passing type information from fix to scan. Currently, the tracer
doesn't offer this option when it's polling for work.
:mps:tag:`scan.stack` Tracing along the reference graph cannot be done by
recursive descent, because we can't guarantee that the stack won't
overflow. We can, however, maintain an explicit stack of things to
trace, and fall back on iterative methods (:mps:ref:`.scan.iter`) when it
overflows and can't be extended.
:mps:tag:`scan.iter` As discussed in :mps:ref:`.scan.segment`, when scanning a
segment, we need to ensure that there are no grey objects in the
segment when the scan method returns. We can do this by iterating a
sequential scan over the segment until nothing is grey (see
:mps:ref:`.marked.scan` for details).
:mps:tag:`scan.iter.only` Some iterative method is needed as a fallback for
the more advanced methods, and as this is the simplest way of
implementing the current tracer protocol, we will start by
implementing it as the only scanning method.
:mps:tag:`scan.buffer` We do not scan between ScanLimit and Limit of a
buffer (see :mps:ref:`.iteration.buffer`), as usual.
.. note::
design.mps.buffer should explain why this works, but doesn't.
Pekka P. Pirinen, 1998-02-11.
:mps:tag:`fix.to-black` When fixing a reference to a white object, if the
segment does not refer to the white set, the object cannot refer to
the white set, and can therefore be marked as black immediately
(rather than grey).
Implementation
--------------
Colour
......
:mps:tag:`colour.determine` Following the plan in :mps:ref:`.colour`, if
``SegWhite(seg)`` includes the trace, the colour of an object is given
by the bit tables. Otherwise if ``SegGrey(seg)`` includes the trace,
all the objects are grey. Otherwise all the objects are black.
:mps:tag:`colour.bits` As we only have searches for runs of zero bits, we use
two bit tables, the non-grey and non-white tables, but this is hidden
beneath a layer of macros talking about grey and white in positive
terms.
:mps:tag:`colour.single` We have only implemented a single set of mark and
scan tables, so we can only condemn a segment for one trace at a time.
This is checked for in condemnation. If we want to do overlapping
white sets, each trace needs its own set of tables.
:mps:tag:`colour.check` The grey-and-non-white state is illegal, and free
objects must be white as explained in
analysis.non-moving-colour.contraint.reclaim.
Iteration
.........
:mps:tag:`iteration` Scan, reclaim and other operations need to iterate over
all objects in a segment. We abstract this into a single iteration
function, even though we no longer use it for reclaiming and rarely
for scanning.
:mps:tag:`iteration.buffer` Iteration skips directly from ScanLimit to Limit
of a buffer. This is because this area may contain
partially-initialized and uninitialized data, which cannot be
processed. Since the iteration skips the buffer, callers need to take
the appropriate action, if any, on it.
.. note::
ScanLimit is used for reasons which are not documented in
design.mps.buffer.
Scanning Algorithm
..................
:mps:tag:`marked` Each segment has a ``marksChanged`` flag, indicating
whether anything in it has been made grey since the last scan
iteration (:mps:ref:`.scan.iter`) started. This flag only concerns the colour
of objects with respect to the trace for which the segment is
condemned, as this is the only trace for which objects in the segment
are being made grey by fixing. Note that this flag doesn't imply that
there are grey objects in the segment, because the grey objects might
have been subsequently scanned and blackened.
:mps:tag:`marked.fix` The ``marksChanged`` flag is set :c:macro:`TRUE` by
:c:func:`AMSFix()` when an object is made grey.
:mps:tag:`marked.scan` :c:func:`AMSScan()` must blacken all grey objects on the
segment, so it must iterate over the segment until all grey objects
have been seen. Scanning an object in the segment might grey another
one (:mps:ref:`.marked.fix`), so the scanner iterates until this flag is
:c:macro:`FALSE`, setting it to :c:macro:`FALSE` before each scan. It is safe to
scan the segment even if it contains nothing grey.
:mps:tag:`marked.scan.fail` If the format scanner returns failure (see
protocol.mps.scanning), we abort the scan in the middle of a segment.
So in this case the marksChanged flag is set back to TRUE, because we
may not have blackened all grey objects.
.. note::
Is that the best reference for the format scanner?
:mps:tag:`marked.unused` The ``marksChanged`` flag is meaningless unless the
segment is condemned. We make it :c:macro:`FALSE` in these circumstances.
:mps:tag:`marked.condemn` Condemnation makes all objects in a segment either
black or white, leaving nothing grey, so it doesn't need to set the
``marksChanged`` flag which must already be :c:macro:`FALSE`.
:mps:tag:`marked.reclaim` When a segment is reclaimed, it can contain
nothing marked as grey, so the ``marksChanged`` flag must already be
:c:macro:`FALSE`.
:mps:tag:`marked.blacken` When the tracer decides not to scan, but to call
:c:func:`PoolBlacken()`, we know that any greyness can be removed.
:c:func:`AMSBlacken()` does this and resets the ``marksChanged`` flag, if it
finds that the segment has been condemned.
:mps:tag:`marked.clever` AMS could be clever about not setting the
``marksChanged`` flag, if the fixed object is ahead of the current
scan pointer. It could also keep low- and high-water marks of grey
objects, but we don't need to implement these improvements at first.
Allocation
..........
:mps:tag:`buffer-init` We take one init arg to set the Rank on the buffer,
just to see how it's done.
:mps:tag:`no-bit` As an optimization, we won't use the alloc bit table until
the first reclaim on the segment. Before that, we just keep a
high-water mark.
:mps:tag:`fill` :c:func:`AMSBufferFill()` takes the simplest approach: it iterates
over the segments in the pool, looking for one which can be used to
refill the buffer.
:mps:tag:`fill.colour` The objects allocated from the new buffer must be
black for all traces (:mps:ref:`.colour.alloc`), so putting it on a black
segment (meaning one where neither ``SegWhite(seg)`` nor
``SegGrey(seg)`` include the trace, see :mps:ref:`.colour.determine`) is
obviously OK. White segments (where ``SegWhite(seg)`` includes the
trace) are also fine, as we can use the colour tables to make it
black. At first glance, it seems we can't put it on a segment that is
grey but not white for some trace (one where ``SegWhite(seg)`` doesn't
include the trace, but ``SegGrey(seg)`` does), because the new objects
would become grey as the buffer's ScanLimit advanced. However, in many
configurations, the mutator would hit a barrier as soon as it started
initializing the object, which would flip the buffer. In fact, the
current (2002-01) implementation of buffers assumes buffers are black,
so they'd better.
:mps:tag:`fill.colour.reclaim` In fact, putting a buffer on a condemned
segment will screw up the accounting in :c:func:`AMCReclaim()`, so it's
disallowed.
:mps:tag:`fill.slow` :c:func:`AMSBufferFill()` gets progressively slower as more
segments fill up, as it laboriously checks whether the buffer can be
refilled from each segment, by inspecting the allocation bit map. This
is helped a bit by keeping count of free grains in each segment, but
it still spends a lot of time iterating over all the full segments
checking the free size. Obviously, this can be much improved (we could
keep track of the largest free block in the segment and in the pool,
or we could keep the segments in some more efficient structure, or we
could have a real free list structure).
:mps:tag:`fill.extend` If there's no space in any existing segment, the
``segSize`` method is called to decide the size of the new segment to
allocate. If that fails, the code tries to allocate a segment that's
just large enough to satisfy the request.
:mps:tag:`empty` :c:func:`AMSBufferEmpty()` makes the unused space free, since
there's no reason not to. We have to adjust the colour tables as well,
since these grains were black and now they need to be white (or at
least encoded -G and W).
:mps:tag:`reclaim.empty.buffer` Segments which after reclaim only contain a
buffer could be destroyed by trapping the buffer, but there's no point
to this.
Initialization
..............
:mps:tag:`init` The initialization method :c:func:`AMSInit()` takes three
additional arguments: the format of objects allocated in the pool, the
chain that controls GC timing, and a flag for supporting ambiguous
references.
:mps:tag:`init.share` If support for ambiguity is required, the
``shareAllocTable`` flag is reset to indicate the pool uses three
separate bit tables, otherwise it is set and the pool shares a table
for non-white and alloc (see :mps:ref:`.colour.encoding`).
:mps:tag:`init.align` The pool alignment is set equal to the format
alignment (see design.mps.align).
:mps:tag:`init.internal` Subclasses call :c:func:`AMSInitInternal()` to avoid the
problems of sharing ``va_list`` and emitting a superfluous
``PoolInitAMS`` event.
Condemnation
............
:mps:tag:`condemn.buffer` Buffers are not condemned, instead they are
coloured black, to make sure that the objects allocated will be black,
following :mps:ref:`.colour.alloc` (or, if you wish, because buffers are
ignored like free space, so need the same encoding).
Reclaim
.......
:mps:tag:`reclaim` Reclaim uses either of analysis.non-moving-colour
.constraint.reclaim.white-free-bit (just reuse the non-white table as
the alloc table) or
analysis.non-moving-colour.constraint.reclaim.free-bit (copy it),
depending on the shareAllocTable flag (as set by :mps:ref:`.init.share`).
However, bit table still has to be iterated over to count the free
grains. Also, in a debug pool, each white block has to be splatted.
Segment merging and splitting
.............................
:mps:tag:`split-merge` We provide methods for splitting and merging AMS
segments. The pool implementation doesn't cause segments to be split
or merged -- but a subclass might want to do this (see
:mps:ref:`.stress.split-merge`). The methods serve as an example of how to
implement this facility.
:mps:tag:`split-merge.constrain` There are some additional constraints on
what segments may be split or merged:
- :mps:tag:`split-merge.constrain.align` Segments may only be split or
merged at an address which is aligned to the pool alignment as well
as to the arena alignment.
:mps:tag:`split-merge.constrain.align.justify` This constraint is implied
by the design of allocation and colour tables, which cannot
represent segments starting at unaligned addresses. The constraint
only arises if the pool alignment is larger than the arena
alignment. There's no requirement to split segments at unaligned
addresses.
- :mps:tag:`split-merge.constrain.empty` The higher segment must be empty.
That is, the higher segment passed to :c:func:`SegMerge()` must be empty,
and the higher segment returned by :c:func:`SegSplit()` must be empty.
:mps:tag:`split-merge.constrain.empty.justify` This constraint makes the
code significantly simpler. There's no requirement for a more
complex solution at the moment (as the purpose is primarily
pedagogic).
:mps:tag:`split-merge.fail` The split and merge methods are not proper
anti-methods for each other (see
design.mps.seg.split-merge.fail.anti.no). Methods will not reverse the
side-effects of their counterparts if the allocation of the colour and
allocation bit tables should fail. Client methods which over-ride
split and merge should not be written in such a way that they might
detect failure after calling the next method, unless they have reason
to know that the bit table allocations will not fail.
Testing
-------
:mps:tag:`stress` There's a stress test, MMsrc!amsss.c, that does 800 kB of
allocation, enough for about three GCs. It uses a modified Dylan
format, and checks for corruption by the GC. Both ambiguous and exact
roots are tested.
:mps:tag:`stress.split-merge` There's also a stress test for segment
splitting and merging, MMsrc!segsmss.c. This is similar to amsss.c --
but it defines a subclass of AMS, and causes segments to be split and
merged. Both buffered and non-buffered segments are split / merged.
Notes
-----
:mps:tag:`addr-index.slow` Translating from an address to and from a grain
index in a segment uses macros such as :c:macro:`AMS_INDEX` and
:c:macro:`AMS_INDEX_ADDR`. These are slow because they call :c:func:`SegBase()` on
every translation -- we could cache that.
:mps:tag:`grey-mutator` To enforce the restriction set in :mps:ref:`.not-req.grey`
we check that all the traces are flipped in :c:func:`AMSScan()`. It would be
good to check in :c:func:`AMSFix()` as well, but we can't do that, because
it's called during the flip, and we can't tell the difference between
the flip and the grey mutator phases with the current tracer
interface.

539
mps/manual/html/_sources/design/poolawl.txt Normal file
View file

@ -0,0 +1,539 @@
.. _design-poolawl:
.. index::
pair: AWL pool class; design
single: pool class; AWL design
AWL pool class
==============
.. mps:prefix:: design.mps.poolawl
pair: AWL pool class; design
single: pool class; AWL design
Introduction
------------
:mps:tag:`readership` Any MPS developer.
:mps:tag:`intro` The AWL (Automatic Weak Linked) pool is used to manage
Dylan Weak Tables (see req.dylan.fun.weak). Currently the design is
specialised for Dylan Weak Tables, but it could be generalised in the
future.
Requirements
------------
See req.dylan.fun.weak.
See meeting.dylan.1997-02-27(0) where many of the requirements for
this pool were first sorted out.
Must satisfy request.dylan.170123.
:mps:tag:`req.obj-format` Only objects of a certain format need be
supported. This format is a subset of the Dylan Object Format. The
pool uses the first slot in the fixed part of an object to store an
association. See mail.drj.1997-03-11.12-05
Definitions
-----------
:mps:tag:`def.grain` alignment grain, grain. A grain is a range of addresses
where both the base and the limit of the range are aligned and the
size of range is equal to the (same) alignment. In this context the
alignment is the pool's alignment (``pool->alignment``). The grain is
the unit of allocation, marking, scanning, etc.
Overview
--------
:mps:tag:`overview`
:mps:tag:`overview.ms` The pool is mark and sweep. :mps:tag:`overview.ms.justify`
Mark-sweep pools are slightly easier to write (than moving pools), and
there are no requirements (yet) that this pool be high performance or
moving or anything like that.
:mps:tag:`overview.alloc` It is possible to allocate weak or exact objects
using the normal reserve/commit AP protocol.
:mps:tag:`overview.alloc.justify` Allocation of both weak and exact objects
is required to implement Dylan Weak Tables. Objects are formatted; the
pool uses format A.
:mps:tag:`overview.scan` The pool handles the scanning of weak objects
specially so that when a weak reference is deleted the corresponding
reference in an associated object is deleted. The associated object is
determined by using information stored in the object itself (see
:mps:ref:`.req.obj-format`).
Interface
---------
:mps:tag:`if.init` The init method takes one extra parameter in the vararg
list. This parameter should have type :c:type:`Format` and be a format
object that describes the format of the objects to be allocated in
this pool. The format should support scan and skip methods. There is
an additional restriction on the layout of objects, see
:mps:ref:`.req.obj-format`.
:mps:tag:`if.buffer` The :c:func:`BufferInit()` method takes one extra parameter
in the vararg list. This parameter should be either ``RankEXACT`` or
``RankWEAK``. It determines the rank of the objects allocated using
that buffer.
Data structures
---------------
:mps:tag:`sig` This signature for this pool will be 0x519bla3l (SIGPooLAWL).
:mps:tag:`poolstruct` The class specific pool structure is::
struct AWLStruct {
PoolStruct poolStruct;
Format format;
Shift alignShift;
ActionStruct actionStruct;
double lastCollected;
Serial gen;
Sig sig;
}
:mps:tag:`poolstruct.format` The format field is used to refer to the object
format. The object format is passed to the pool during pool creation.
:mps:tag:`poolstruct.alignshift` The ``alignShift`` field is the
``SizeLog2`` of the pool's alignment. It is computed and initialised
when a pool is created. It is used to compute the number of alignment
grains in a segment which is the number of bits need in the segment's
mark and alloc bit table (see :mps:ref:`.awlseg.bt`, :mps:ref:`.awlseg.mark`, and
:mps:ref:`.awlseg.alloc` below).
.. note::
Clarify this.
:mps:tag:`poolstruct.actionStruct` Contains an Action which is used to
participate in the collection benefit protocol. See :c:func:`AWLBenefit()`
below for a description of the algorithm used for determining when to
collect.
:mps:tag:`poolstruct.lastCollected` Records the time (using the mutator
total allocation clock, ie that returned by
:c:func:`ArenaMutatorAllocSize()`) of the most recent call to either
:c:func:`AWLInit()` or :c:func:`AWLTraceBegin()` for this pool. So this is the
time of the beginning of the last collection of this pool. Actually
this isn't true because the pool can be collected without
:c:func:`AWLTraceBegin()` being called (I think) as it will get collected by
being in the same zone as another pool/generation that is being
collected (which it does arrange to be, see the use of the gen field
in :mps:ref:`.poolstruct.gen` below and :mps:ref:`.fun.awlsegcreate.where` below).
:mps:tag:`poolstruct.gen` This part of the mechanism by which the pool
arranges to be in a particular zone and arranges to be collected
simultaneously with other cohorts in the system. ``gen`` is the
generation that is used in expressing a generation preference when
allocating a segment. The intention is that this pool will get
collected simultaneously with any other segments that are also
allocated using this generation preference (when using the VM arena,
generation preferences get mapped more or less to zones, each
generation to a unique set of zones in the ideal case). Whilst AWL is
not generational it is expected that this mechanism will arrange for
it to be collected simultaneously with some particular generation of
AMC.
:mps:tag:`poolstruct.gen.1` At the moment the ``gen`` field is set for all
AWL pools to be 1.
:mps:tag:`awlseg` The pool defines a segment class :c:type:`AWLSegClass`, which is
a subclass of :c:type:`GCSegClass` (see
design.mps.seg.over.hierarchy.gcseg). All segments allocated by the
pool are instances of this class, and are of type ``AWLSeg``, for
which the structure is::
struct AWLSegStruct {
GCSegStruct gcSegStruct;
BT mark;
BT scanned;
BT alloc;
Count grains;
Count free;
Count singleAccesses;
AWLStatSegStruct stats;
Sig sig;
}
:mps:tag:`awlseg.bt` The mark, alloc, and scanned fields are bit-tables (see
design.mps.bt). Each bit in the table corresponds to a a single
alignment grain in the pool.
:mps:tag:`awlseg.mark` The mark bit table is used to record mark bits during
a trace. :c:func:`AWLCondemn()` (see :mps:ref:`.fun.condemn` below) sets all the
bits of this table to zero. Fix will read and set bits in this table.
Currently there is only one mark bit table. This means that the pool
can only be condemned for one trace.
:mps:tag:`awlseg.mark.justify` This is simple, and can be improved later
when we want to run more than one trace.
:mps:tag:`awlseg.scanned` The scanned bit-table is used to note which
objects have been scanned. Scanning (see :mps:ref:`.fun.scan` below) a segment
will find objects that are marked but not scanned, scan each object
found and set the corresponding bits in the scanned table.
:mps:tag:`awlseg.alloc` The alloc bit table is used to record which portions
of a segment have been allocated. Ranges of bits in this table are set
when a buffer is attached to the segment. When a buffer is flushed (ie
:c:func:`AWLBufferEmpty()` is called) from the segment, the bits
corresponding to the unused portion at the end of the buffer are
reset.
:mps:tag:`awlseg.alloc.invariant` A bit is set in the alloc table if and
only if the corresponding address is currently being buffered, or the
corresponding address lies within the range of an allocated object.
:mps:tag:`awlseg.grains` The grains field is the number of grains that fit
in the segment. Strictly speaking this is not necessary as it can be
computed from ``SegSize`` and AWL's alignment, however, precalculating
it and storing it in the segment makes the code simpler by avoiding
lots of repeated calculations.
:mps:tag:`awlseg.free` A conservative estimate of the number of free grains
in the segment. It is always guaranteed to be greater than or equal to
the number of free grains in the segment, hence can be used during
allocation to quickly pass over a segment.
.. note::
Maintained by blah and blah. Unfinished obviously.
Functions
---------
.. note::
How will pool collect? It needs an action structure.
External
........
.. c:function:: Res AWLInit(Pool pool, va_list arg)
:mps:tag:`fun.init` :c:type:`AWLStruct` has four fields, each one needs initializing.
:mps:tag:`fun.init.poolstruct` The ``poolStruct`` field has already been
initialized by generic code (impl.c.pool).
:mps:tag:`fun.init.format` The format will be copied from the argument list,
checked, and written into this field.
:mps:tag:`fun.init.alignshift` The ``alignShift`` will be computed from the
pool alignment and written into this field.
:mps:tag:`fun.init.sig` The ``sig`` field will be initialized with the
signature for this pool.
.. c:function:: Res AWLFinish(Pool pool)
:mps:tag:`fun.finish` Iterates over all segments in the pool and destroys
each segment (by calling :c:func:`SegFree()`). Overwrites the sig field in
the :c:type:`AWLStruct`. Finishing the generic pool structure is done by the
generic pool code (impl.c.pool).
:mps:tag:`fun.alloc` :c:func:`PoolNoAlloc()` will be used, as this class does not
implement alloc.
:mps:tag:`fun.free` :c:func:`PoolNoFree()` will be used, as this class does not
implement free.
.. c:function:: Res AWLBufferFill(Seg *segReturn, Addr *baseReturn, Pool pool, Buffer buffer, Size size)
:mps:tag:`fun.fill` This zips round all the the segments applying
:c:func:`AWLSegAlloc()` to each segment that has the same rank as the
buffer. :c:func:`AWLSegAlloc()` attempts to find a free range, if it finds a
range then it may be bigger than the actual request, in which case the
remainder can be used to "fill" the rest of the buffer. If no free
range can be found in an existing segment then a new segment will be
created (which is at least large enough). The range of buffered
addresses is marked as allocated in the segment's alloc table.
.. c:function:: void AWLBufferEmpty(Pool pool, Buffer buffer)
:mps:tag:`fun.empty` Locates the free portion of the buffer, that is the
memory between the init and the limit of the buffer and records these
locations as being free in the relevant alloc table. The segment that
the buffer is pointing at (which contains the alloc table that needs
to be dinked with) is available via :c:func:`BufferSeg()`.
:mps:tag:`fun.benefit` The benefit returned is the total amount of mutator
allocation minus the ``lastRembemberedSize`` minus 10 MiB, so the pool
becomes an increasingly good candidate for collection at a constant
(mutator allocation) rate, crossing the 0 line when there has been
10 MiB of allocation since the (beginning of the) last collection. So
it gets collected approximately every 10 MiB of allocation. Note that
it will also get collected by virtue of being in the same zone as some
AMC generation (assuming there are instantiated AMC pools), see
:mps:ref:`.poolstruct.gen` above.
.. c:function:: Res AWLCondemn(Pool pool, Trace trace, Seg seg)
:mps:tag:`fun.condemn` The current design only permits each segment to be
condemned for one trace (see :mps:ref:`.awlseg.mark`). This function checks
that the segment is not condemned for any trace (``seg->white ==
TraceSetEMPTY``). The segment's mark bit-table is reset, and the
whiteness of the seg (``seg->white``) has the current trace added to
it.
.. c:function:: void AWLGrey(Pool pool, Trace trace, Seg seg)
:mps:tag:`fun.grey` If the segment is not condemned for this trace the
segment's mark table is set to all 1s and the segment is recorded as
being grey.
.. c:function:: Res AWLScan(ScanState ss, Pool pool, Seg seg)
:mps:tag:`fun.scan`
:mps:tag:`fun.scan.overview` The scanner performs a number of passes over
the segment, scanning each marked and unscanned (grey) object that is
finds.
:mps:tag:`fun.scan.overview.finish` It keeps perform a pass over the segment
until it is finished.
:mps:tag:`fun.scan.overview.finish.condition` A condition for finishing is
that no new marks got placed on objects in this segment during the
pass.
:mps:tag:`fun.scan.overview.finish.approximation` We use an even stronger
condition for finishing that assumes that scanning any object may
introduce marks onto this segment. It is finished when a pass results
in scanning no objects (that is, all objects were either unmarked or
both marked and scanned).
:mps:tag:`fun.scan.overview.finished-flag` There is a flag called
``finished`` which keeps track of whether we should finish or not. We
only ever finish at the end of a pass. At the beginning of a pass the
flag is set. During a pass if any objects are scanned then the
``finished`` flag is reset. At the end of a pass if the ``finished``
flag is still set then we are finished. No more passes take place and
the function returns.
:mps:tag:`fun.scan.pass` A pass consists of a setup phase and a repeated
phase.
:mps:tag:`fun.scan.pass.buffer` The following assumes that in the general
case the segment is buffered; if the segment is not buffered then the
actions that mention buffers are not taken (they are unimportant if
the segment is not buffered).
:mps:tag:`fun.scan.pass.p` The pass uses a cursor called ``p`` to progress
over the segment. During a pass ``p`` will increase from the base
address of the segment to the limit address of the segment. When ``p``
reaches the limit address of the segment, the pass in complete.
:mps:tag:`fun.scan.pass.setup` ``p`` initially points to the base address of
the segment.
:mps:tag:`fun.scan.pass.repeat` The following comprises the repeated phase.
The repeated phase is repeated until the pass completion condition is
true (that is, ``p`` has reached the limit of the segment, see
:mps:ref:`.fun.scan.pass.p` above and :mps:ref:`.fun.scan.pass.repeat.complete`
below).
:mps:tag:`fun.scan.pass.repeat.complete` If ``p`` is equal to the segment's
limit then we are done. We proceed to check whether any further passes
need to be performed (see :mps:ref:`.fun.scan.pass.more` below).
:mps:tag:`fun.scan.pass.repeat.free` If ``!alloc(p)`` (the grain is free)
then increment ``p`` and return to the beginning of the loop.
:mps:tag:`fun.scan.pass.repeat.buffer` If ``p`` is equal to the buffer's
ScanLimit, as returned by :c:func:`BufferScanLimit()`, then set ``p`` equal
to the buffer's Limit, as returned by :c:func:`BufferLimit()` and return to
the beginning of the loop.
:mps:tag:`fun.scan.pass.repeat.object-end` The end of the object is located
using the ``format->skip`` method.
:mps:tag:`fun.scan.pass.repeat.object` if ``mark(p) && !scanned(p)`` then
the object pointed at is marked but not scanned, which means we must
scan it, otherwise we must skip it.
:mps:tag:`fun.scan.pass.repeat.object.dependent` To scan the object the
object we first have to determine if the object has a dependent object (see
:mps:ref:`.req.obj-format`).
:mps:tag:`fun.scan.pass.repeat.object.dependent.expose` If it has a
dependent object then we must expose the segment that the dependent
object is on (only if the dependent object actually points to MPS
managed memory) prior to scanning and cover the segment subsequent to
scanning.
:mps:tag:`fun.scan.pass.repeat.object.dependent.summary` The summary of the
dependent segment must be set to ``RefSetUNIV`` to reflect the fact
that we are allowing it to be written to (and we don't know what gets
written to the segment).
:mps:tag:`fun.scan.pass.repeat.object.scan` The object is then scanned by
calling the format's scan method with base and limit set to the
beginning and end of the object (:mps:tag:`fun.scan.scan.improve.single` A
scan1 format method would make it slightly simpler here). Then the
finished flag is cleared and the bit in the segment's scanned table is
set.
:mps:tag:`fun.scan.pass.repeat.advance` ``p`` is advanced past the object
and we return to the beginning of the loop.
:mps:tag:`fun.scan.pass.more` At the end of a pass the finished flag is
examined.
:mps:tag:`fun.scan.pass.more.not` If the finished flag is set then we are
done (see :mps:ref:`.fun.scan.overview.finished-flag` above), :c:func:`AWLScan()`
returns.
:mps:tag:`fun.scan.pass.more.so` Otherwise (the finished flag is reset) we
perform another pass (see :mps:ref:`.fun.scan.pass` above).
.. c:function:: Res AWLFix(Pool pool, ScanState ss, Seg seg, Ref *refIO)
:mps:tag:`fun.fix` ``ss->wasMarked`` is set to :c:macro:`TRUE` (clear compliance
with design.mps.fix.protocol.was-marked.conservative).
If the rank (``ss->rank``) is ``RankAMBIG`` then fix returns
immediately unless the reference is aligned to the pool alignment.
If the rank (``ss->rank``) is ``RankAMBIG`` then fix returns
immediately unless the referenced grain is allocated.
The bit in the marked table corresponding to the referenced grain will
be read. If it is already marked then fix returns. Otherwise (the
grain is unmarked), ``ss->wasMarked`` is set to :c:macro:`FALSE`, the
remaining actions depend on whether the rank (``ss->rank``) is
``RankWEAK`` or not. If the rank is weak then the reference is
adjusted to 0 (see design.mps.weakness) and fix returns. If the rank
is something else then the mark bit corresponding to the referenced
grain is set, and the segment is greyed using :c:func:`TraceSegGreyen()`.
Fix returns.
.. c:function:: void AWLReclaim(Pool pool, Trace trace, Seg seg)
:mps:tag:`fun.reclaim` This iterates over all allocated objects in the
segment and frees objects that are not marked. When this iteration is
complete the marked array is completely reset.
``p`` points to base of segment. Then::
while(p < SegLimit(seg) {
if(!alloc(p)) { ++p;continue; }
q = skip(p) /* q points to just past the object pointed at by p */
if !marked(p) free(p, q); /* reset the bits in the alloc table from p to q-1 inclusive. */
p = q
}
Finally, reset the entire marked array using :c:func:`BTResRange()`.
:mps:tag:`fun.reclaim.improve.pad` Consider filling free ranges with padding
objects. Now reclaim doesn't need to check that the objects are
allocated before skipping them. There may be a corresponding change
for scan as well.
.. c:function:: Res AWLDescribe(Pool pool, mps_lib_FILE *stream)
:mps:tag:`fun.describe`
Internal
........
.. c:function:: Res AWLSegCreate(AWLSeg *awlsegReturn, Size size)
:mps:tag:`fun.awlsegcreate` Creates a segment of class :c:type:`AWLSegClass` of size at least ``size``.
:mps:tag:`fun.awlsegcreate.size.round` ``size`` is rounded up to an
``ArenaAlign`` before requesting the segment.
:mps:tag:`fun.awlsegcreate.size.round.justify` The arena requires that all
segment sizes are aligned to the ``ArenaAlign``.
:mps:tag:`fun.awlsegcreate.where` The segment is allocated using a
generation preference, using the generation number stored in the
:c:type:`AWLStruct` (the ``gen`` field), see :mps:ref:`.poolstruct.gen` above.
.. c:function:: Res awlSegInit(Seg seg, Pool pool, Addr base, Size size, Bool reservoirPermit, va_list args)
:mps:tag:`fun.awlseginit` Init method for :c:type:`AWLSegClass`, called for
:c:func:`SegAlloc()` whenever an ``AWLSeg`` is created (see
:mps:ref:`.fun.awlsegcreate` above).
:mps:tag:`fun.awlseginit.tables` The segment's mark scanned and alloc tables
(see :mps:ref:`.awlseg.bt` above) are allocated and initialised. The segment's
grains field is computed and stored.
.. c:function:: void awlSegFinish(Seg seg)
:mps:tag:`fun.awlsegfinish` Finish method for :c:type:`AWLSegClass`, called from
:c:func:`SegFree()`. Will free the segment's tables (see :mps:ref:`.awlseg.bt`).
.. c:function:: Bool AWLSegAlloc(Addr *baseReturn, Addr *limitReturn, AWLSeg awlseg, AWL awl, Size size)
:mps:tag:`fun.awlsegalloc` Will search for a free block in the segment that
is at least size bytes long. The base address of the block is returned
in ``*baseReturn``, the limit of the entire free block (which must be
at least as large size and may be bigger) is returned in
``*limitReturn``. The requested size is converted to a number of
grains, :c:func:`BTFindResRange()` is called to find a run of this length in
the alloc bit-table (:mps:ref:`.awlseg.alloc`). The return results (if it is
successful) from :c:func:`BTFindResRange()` are in terms of grains, they are
converted back to addresses before returning the relevant values from
this function.
.. c:function:: Bool AWLDependentObject(Addr *objReturn, Addr parent)
:mps:tag:`fun.dependent-object` This function abstracts the association
between an object and its linked dependent (see :mps:ref:`.req.obj-format`).
It currently assumes that objects are Dylan Object formatted according
to design.dylan.container (see analysis.mps.poolawl.dependent.abstract
for suggested improvements). An object has a dependent object iff the
second word of the object, that is, ``((Word *)parent)[1]``, is
non-:c:macro:`NULL`. The dependent object is the object referenced by the
second word and must be a valid object.
This function assumes objects are in Dylan Object Format (see
design.dylan.container). It will check that the first word looks like
a Dylan wrapper pointer. It will check that the wrapper indicates that
the wrapper has a reasonable format (namely at least one fixed field).
If the second word is :c:macro:`NULL` it will return :c:macro:`FALSE`. If the second
word is non-:c:macro:`NULL` then the contents of it will be assigned to
``*objReturn``, and it will return :c:macro:`TRUE`.
Test
----
- must create Dylan objects.
- must create Dylan vectors with at least one fixed field.
- must allocate weak thingies.
- must allocate exact tables.
- must link tables together.
- must populate tables with junk.
- some junk must die.
Use an LO pool and an AWL pool. Three buffers. One buffer for the LO
pool, one exact buffer for the AWL pool, one weak buffer for the AWL
pool.
Initial test will allocate one object from each buffer and then
destroy all buffers and pools and exit

249
mps/manual/html/_sources/design/poollo.txt Normal file
View file

@ -0,0 +1,249 @@
.. _design-poollo:
.. index::
pair: LO pool class; design
single: pool class; LO design
LO pool class
=============
.. mps:prefix:: design.mps.poollo
pair: LO pool class; design
single: pool class; LO design
Introduction
------------
:mps:tag:`readership` Any MPS developer.
:mps:tag:`intro` The LO (Leaf Object) pool class is a pool class developed
for DylanWorks. It is designed to manage objects that have no
references (leaf objects) such as strings, bit tables, etc. It is a
garbage collected pool (in that objects allocated in the pool are
automatically reclaimed when they are discovered to be unreachable.
.. note::
Need to sort out issue of alignment. Currently lo grabs alignment
from format, almost certainly "ought" to use the greater of the
format alignment and the :c:macro:`MPS_ALIGN` value. David Jones,
1997-07-02.
Definitions
-----------
:mps:tag:`def.leaf` A "leaf" object is an object that contains no
references, or an object all of whose references refer to roots. That
is, any references that the object has must refer to a priori alive
objects that are guaranteed not to move, hence the references do not
need fixing.
:mps:tag:`def.grain` A grain (of some alignment) is a contiguous aligned
area of memory of the smallest size possible (which is the same size
as the alignment).
Requirements
------------
:mps:tag:`req.source` See req.dylan.fun.obj.alloc and
req.dylan.prot.ffi.access.
:mps:tag:`req.leaf` The pool must manage formatted leaf objects (see
:mps:ref:`.def.leaf` above for a definition). This is intended to encompass
Dylan and C leaf objects. Dylan leaf objects have a reference to their
wrapper, but are still leaf objects (in the sense of :mps:ref:`.def.leaf`)
because the wrapper will be a root.
:mps:tag:`req.nofault` The memory containing objects managed by the pool
must not be protected. The client must be allowed to access these
objects without using the MPS trampoline (the exception mechanism,
q.v.).
Overview
--------
:mps:tag:`overview`
:mps:tag:`overview.ms` The LO Pool is a non-moving mark-and-sweep collector.
:mps:tag:`overview.ms.justify` Mark-and-sweep pools are simpler than moving
pools.
:mps:tag:`overview.alloc` Objects are allocated in the pool using the
reserve/commit protocol on allocation points.
:mps:tag:`overview.format` The pool is formatted. The format of the objects
in the pool is specified at instantiation time, using an format object
derived from a variant A format (using variant A is overkill, see
:mps:ref:`.if.init` below) (see design.mps.format for excuse about calling the
variant 'A').
Interface
---------
:mps:tag:`if.init`
:mps:tag:`if.init.args` The init method for this class takes one extra
parameter in the vararg parameter list.
:mps:tag:`if.init.format` The extra parameter should be an object of type
Format and should describe the format of the objects that are to be
allocated in the pool.
:mps:tag:`if.init.format.use` The pool uses the skip and alignment slots of
the format. The skip method is used to determine the length of objects
(during reclaim). The alignment field is used to determine the
granularity at which memory should be managed.
:mps:tag:`if.init.format.a` Currently only format variant A is supported
though clearly that is overkill as only skip and alignment are used.
Data structures
---------------
:mps:tag:`sig` The signature for the LO Pool Class is 0x51970b07
(SIGLOPOoL).
:mps:tag:`poolstruct` The class specific pool structure is::
typedef struct LOStruct {
PoolStruct poolStruct; /* generic pool structure */
Format format; /* format for allocated objects */
Shift alignShift;
Sig sig; /* impl.h.misc.sig */
} LOStruct;
:mps:tag:`poolstruct.format` This is the format of the objects that are
allocated in the pool.
:mps:tag:`poolstruct.alignShift` This is shift used in alignment
computations. It is ``SizeLog2(pool->alignment).`` It can be used on
the right of a shift operator (``<<`` or ``>>``) to convert between a
number of bytes and a number of grains.
:mps:tag:`loseg` Every segment is an instance of segment class :c:type:`LOSegClass`, a
subclass of :c:type:`GCSegClass`, and is an object of type :c:type:`LOSegStruct`.
:mps:tag:`loseg.purpose` The purpose of the ``LOSeg`` structure is to
associate the bit tables used for recording allocation and mark
information with the segment.
:mps:tag:`loseg.decl` The declaration of the structure is as follows::
typedef struct LOSegStruct {
GCSegStruct gcSegStruct; /* superclass fields must come first */
LO lo; /* owning LO */
BT mark; /* mark bit table */
BT alloc; /* alloc bit table */
Count free; /* number of free grains */
Sig sig; /* impl.h.misc.sig */
} LOSegStruct;
:mps:tag:`loseg.sig` The signature for a loseg is 0x519705E9 (SIGLOSEG).
:mps:tag:`loseg.lo` The lo field points to the LO structure that owns this
segment.
:mps:tag:`loseg.bit` Bit Tables (see design.mps.bt) are used to record
allocation and mark information. This is relatively straightforward,
but might be inefficient in terms of space in some circumstances.
:mps:tag:`loseg.mark` This is a Bit Table that is used to mark objects
during a trace. Each grain in the segment is associated with 1 bit in
this table. When :c:func:`LOFix()` (see :mps:ref:`.fun.fix` below) is called the
address is converted to a grain within the segment and the
corresponding bit in this table is set.
:mps:tag:`loseg.alloc` This is a Bit Table that is used to record which
addresses are allocated. Addresses that are allocated and are not
buffered have their corresponding bit in this table set. If a bit in
this table is reset then either the address is free or is being
buffered.
:mps:tag:`loseg.diagram` The following diagram is now obsolete. It's also
not very interesting - but I've left the sources in case anyone ever
gets around to updating it. tony 1999-12-16
[missing diagram]
Functions
---------
External
........
:mps:tag:`fun.init`
:mps:tag:`fun.destroy`
:mps:tag:`fun.buffer-fill`
.. note::
Explain way in which buffers interact with the alloc table and how
it could be improved.
:mps:tag:`fun.buffer-empty`
:mps:tag:`fun.condemn`
.. c:function:: Res LOFix(Pool pool, ScanState ss, Seg seg, Ref *refIO)
:mps:tag:`fun.fix` Fix treats references of most ranks much the same. There
is one mark table that records all marks. A reference of rank
``RankAMBIG`` is first checked to see if it is aligned to the pool
alignment and discarded if not. The reference is converted to a grain
number within the segment (by subtracting the segments' base from the
reference and then dividing by the grain size). The bit (the one
corresponding to the grain number) is set in the mark table.
Exception, for a weak reference (rank is ``RankWEAK``) the mark table
is checked and the reference is fixed to 0 if this address has not
been marked otherwise nothing happens. Note that there is no check
that the reference refers to a valid object boundary (which wouldn't
be a valid check in the case of ambiguous references anyway).
.. c:function:: void LOReclaim(Pool pool, Trace trace, Seg seg)
:mps:tag:`fun.reclaim` Derives the loseg from the seg, and calls
:c:func:`loSegReclaim()` (see :mps:ref:`.fun.segreclaim` below).
Internal
........
.. c:function:: void loSegReclaim(LOSeg loseg, Trace trace)
:mps:tag:`fun.segreclaim` For all the contiguous allocated regions in the
segment it locates the boundaries of all the objects in that region by
repeatedly skipping (by calling ``format->skip``) from the beginning
of the region (the beginning of the region is guaranteed to coincide
with the beginning of an object). For each object it examines the bit
in the mark bit table that corresponds to the beginning of the object.
If that bit is set then the object has been marked as a result of a
previous call to :c:func:`LOFix()`, the object is preserved by doing
nothing. If that bit is not set then the object has not been marked
and should be reclaimed; the object is reclaimed by resetting the
appropriate range of bits in the segment's free bit table.
.. note::
Special things happen for buffered segments.
Explain how the marked variable is used to free segments.
Attachment
----------
[missing attachment "LOGROUP.CWK"]

30
mps/manual/html/_sources/design/poolmfs.txt Normal file
View file

@ -0,0 +1,30 @@
.. _design-poolmfs:
.. index::
pair: MFS pool class; design
single: pool class; MFS design
MFS pool class
==============
.. mps:prefix:: design.mps.poolmfs
pair: MFS pool class; design
single: pool class; MFS design
Overview
--------
MFS stands for "Manual Fixed Small". The MFS pool class manages
objects that are of a fixed size. It is intended to only manage small
objects efficiently. Storage is recycled manually by the client
programmer.
A particular instance of an MFS Pool can manage objects only of a
single size, but different instances can manage objects of different
sizes. The size of object that an instance can manage is declared when
the instance is created.

659
mps/manual/html/_sources/design/poolmrg.txt Normal file
View file

@ -0,0 +1,659 @@
.. _design-poolmrg:
.. index::
pair: MRG pool class; design
single: pool class; MRG design
MRG pool class
==============
.. mps:prefix:: design.mps.poolmrg
pair: MRG pool class; design
single: pool class; MRG design
Introduction
------------
:mps:tag:`readership` Any MPS developer.
:mps:tag:`intro` This is the design of the MRG (Manual Rank Guardian) pool
class. The MRG pool class is part of the MPS. The MRG pool class is
internal to the MPS (has no client interface) and is used to implement
finalization.
:mps:tag:`source` Some of the techniques in paper.dbe93 ("Guardians in a
Generation-Based Garbage Collector") were used in this design. Some
analysis of this design (including various improvements and some more
in-depth justification) is in analysis.mps.poolmrg. That document
should be understood before changing this document. It is also helpful
to look at design.mps.finalize and design.mps.message.
Goals
-----
:mps:tag:`goal.final` The MRG pool class should support all
requirements pertaining to finalization.
Requirements
------------
:mps:tag:`req` We have only one requirement pertaining to finalization:
:mps:tag:`req.dylan.fun.finalization` Support the Dylan language-level
implementation of finalized objects: objects are registered, and are
finalized in random order when they would otherwise have died. Cycles
are broken at random places. There is no guarantee of promptness.
:mps:tag:`req.general` However, finalization is a very common piece of
functionality that is provided by (sophisticated) memory managers, so
we can expect other clients to request this sort of functionality.
:mps:tag:`anti-req` Is it required that the MRG pool class return
unused segments to the arena? MFS, for example, does not do this. MRG
will not do this in its initial implementation.
Terminology
-----------
:mps:tag:`def.mrg` **MRG**: The MRG pool class's identifier will be MRG.
This stands for "Manual Rank Guardian". The pool is manually managed
and implements guardians for references of a particular rank
(currently just final).
:mps:tag:`def.final.ref` **final reference**: A reference of rank final (see
design.mps.type.rank).
:mps:tag:`def.final.object` **finalizable object**: An object is finalizable
with respect to a final reference if, since the creation of that
reference, there was a point in time when no references to the object
of lower (that is, stronger) rank were reachable from a root.
:mps:tag:`def.final.object.note` Note that this means an object can be
finalizable even if it is now reachable from the root via exact
references.
:mps:tag:`def.finalize` **finalize**: To finalize an object is to notify the
client that the object is finalizable. The client is presumed to be
interested in this information (typically it will apply some method to
the object).
:mps:tag:`def.guardian` **guardian**: An object allocated in the MRG
Pool. A guardian contains exactly one final reference, and some fields
for the pool's internal use. Guardians are used to implement a
finalization mechanism.
Overview
--------
:mps:tag:`over` The MRG pool class is a pool class in the MPS. It is
intended to provide the functionality of "finalization".
:mps:tag:`over.internal` The MRG pool class is internal to the MPM: it
is not intended to have a client interface. Clients are expected to
access the functionality provided by this pool (finalization) using a
separate MPS finalization interface (design.mps.finalize).
:mps:tag:`over.one-size` The MRG pool class manages objects of a single
size, each object containing a single reference of rank final.
:mps:tag:`over.one-size.justify` This is all that is necessary to meet our
requirements for finalization. Whenever an object is registered for
finalization, it is sufficient to create a single reference of rank
final to it.
:mps:tag:`over.queue` A pool maintains a list of live guardian objects,
called (for historical reasons) the "entry" list.
:mps:tag:`over.queue.free` The pool also maintains a list of free guardian
objects called the "free" list.
:mps:tag:`over.queue.exit.not` There used to be an "exit" list, but this is
now historical and there shouldn't be any current references to it.
:mps:tag:`over.alloc` When guardians are allocated, they are placed on the
entry list. Guardians on the entry list refer to objects that have not
yet been shown to be finalizable (either the object has references of
lower rank than final to it, or the MPS has not yet got round to
determining that the object is finalizable).
:mps:tag:`over.message.create` When a guardian is discovered to refer to a
finalizable object it is removed from the entry list and becomes a
message on the arena's messages queue.
:mps:tag:`over.message.deliver` When the MPS client receives the message the
message system arranges for the message to be destroyed and the pool
reclaims the storage associated with the guardian/message.
:mps:tag:`over.scan` When the pool is scanned at rank final each reference
will be fixed. If the reference is to an unmarked object (before the
fix), then the object must now be finalizable. In this case the
containing guardian will be removed from the entry list and posted as
a message.
:mps:tag:`over.scan.justify` The scanning process is a crucial step
necessary for implementing finalization. It is the means by which the
MPS detects that objects are finalizable.
:mps:tag:`over.message` ``PoolClassMRG`` implements a :c:type:`MessageClass` (see
design.mps.message). All the messages are of one ``MessageType``. This
type is ``MessageTypeFinalization``. Messages are created when objects
are discovered to be finalizable and destroyed when the MPS client has
received the message.
:mps:tag:`over.message.justify` Messages provide a means for the MPS to
communicate with its client. Notification of finalization is just such
a communication. Messages allow the MPS to inform the client of
finalization events when it is convenient for the MPS to do so (i.e.
not in PageFault context).
:mps:tag:`over.manual` Objects in the MRG pool are manually managed.
:mps:tag:`over.manual.alloc` They are allocated by :c:func:`ArenaFinalize()` when
objects are registered for finalization.
:mps:tag:`over.manual.free` They are freed when the associated message is
destroyed.
:mps:tag:`over.manual.justify` The lifetime of a guardian object is very
easy to determine so manual memory management is appropriate.
Protocols
---------
Object Registration
...................
:mps:tag:`protocol.register` There is a protocol by which objects can be
registered for finalization. This protocol is handled by the arena
module on behalf of finalization. see
design.mps.finalize.int.finalize.
Finalizer execution
...................
:mps:tag:`protocol.finalizer` If an object is proven to be finalizable then
a message to this effect will eventually be posted. A client can
receive the message, determine what to do about it, and do it.
Typically this would involve calling the finalization method for the
object, and deleting the message. Once the message is deleted, the
object may become recyclable.
Setup / destroy
...............
:mps:tag:`protocol.life` An instance of PoolClassMRG is needed in order to
support finalization, it is called the "final" pool and is attached to
the arena (see design.mps.finalize.int.arena.struct).
:mps:tag:`protocol.life.birth` The final pool is created lazily by
:c:func:`ArenaFinalize()`.
:mps:tag:`protocol.life.death` The final pool is destroyed during
:c:func:`ArenaDestroy()`.
Data structures
---------------
:mps:tag:`guardian` The guardian
:mps:tag:`guardian.over` A guardian is an object used to manage the
references and other data structures that are used by the pool in
order to keep track of which objects are registered for finalization,
which ones have been finalized, and so on.
:mps:tag:`guardian.state` A guardian can be in one of four states:
:mps:tag:`guardian.state.enum` The states are Free, Prefinal, Final,
PostFinal (referred to as MRGGuardianFree, etc. in the
implementation).
1. :mps:tag:`guardian.state.free` The guardian is free, meaning that it is
on the free list for the pool and available for allocation.
2. :mps:tag:`guardian.state.prefinal` The guardian is allocated, and refers
to an object that has not yet been discovered to be finalizable. It
is on the entry list for the pool.
3. :mps:tag:`guardian.state.final` The guardian is allocated, and refers to
an object that has been shown to be finalizable; this state
corresponds to the existence of a message.
4. :mps:tag:`guardian.state.postfinal` This state is only used briefly and
is entirely internal to the pool; the guardian enters this state
just after the associated message has been destroyed (which happens
when the client receives the message) and will be freed immediately
(whereupon it will enter the Free state). This state is used for
checking only (so that MRGFree can check that only guardians in
this state are being freed).
:mps:tag:`guardian.life-cycle` Guardians go through the following state life-cycle: Free ⟶ Prefinal ⟶ Final ⟶ Postfinal ⟶ Free.
:mps:tag:`guardian.two-part` A guardian is a structure consisting abstractly
of a link part and a reference part. Concretely, the link part is a
:c:type:`LinkPartStruct`, and the reference part is a :c:type:`RefPartStruct`
(which is just a :c:type:`Word`). The link part is used by the pool, the
reference part forms the object visible to clients of the pool. The
reference part is the reference of ``RankFINAL`` that refers to
objects registered for finalization and is how the MPS detects
finalizable objects.
:mps:tag:`guardian.two-part.union` The :c:type:`LinkPartStruct` is a discriminated
union of a :c:type:`RingStruct` and a :c:type:`MessageStruct`. The :c:type:`RingStruct`
is used when the guardian is either Free or Prefinal. The
MessageStruct is used when the guardian is Final. Neither part of the
union is used when the guardian is in the Postfinal state.
:mps:tag:`guardian.two-part.justify` This may seem a little profligate with
space, but this is okay as we are not required to make finalization
extremely space efficient.
:mps:tag:`guardian.parts.separate` The two parts will be stored in separate
segments.
:mps:tag:`guardian.parts.separate.justify` This is so that the data
structures the pool uses to manage the objects can be separated from
the objects themselves. This avoids the pool having to manipulate data
structures that are on shielded segments
(analysis.mps.poolmrg.hazard.shield).
:mps:tag:`guardian.assoc` Ref part number *n* (from the beginning of the
segment) in one segment will correspond with link part number *n* in
another segment. The association between the two segments will be
managed by the additional fields in pool-specific segment subclasses
(see :mps:ref:`.mrgseg`).
:mps:tag:`guardian.ref` Guardians that are either Prefinal or Final are live
and have valid references (possibly :c:macro:`NULL`) in their ref parts.
Guardians that are free are dead and always have :c:macro:`NULL` in their ref
parts (see :mps:ref:`.free.overwrite` and :mps:ref:`.scan.free`).
:mps:tag:`guardian.ref.free` When freeing an object, it is a pointer to the
reference part that will be passed (internally in the pool).
:mps:tag:`guardian.init` Guardians are initialized when the pool is grown
(:mps:ref:`.alloc.grow`). The initial state has the ref part :c:macro:`NULL` and the
link part is attached to the free ring. Freeing an object returns a
guardian to its initial state.
:mps:tag:`poolstruct` The Pool structure, :c:type:`MRGStruct` will have:
- :mps:tag:`poolstruct.entry` the head of the entry list.
- :mps:tag:`poolstruct.free` the head of the free list.
- :mps:tag:`poolstruct.rings` The entry list, the exit list, and the free
list will each be implemented as a :c:type:`Ring`. Each ring will be
maintained using the link part of the guardian.
:mps:tag:`poolstruct.rings.justify` This is because rings are convenient to
use and are well tested. It is possible to implement all three lists
using a singly linked list, but the saving is certainly not worth
making at this stage.
- :mps:tag:`poolstruct.refring` a ring of "ref" segments in use for links or
messages (see .mrgseg.ref.mrgring below).
- :mps:tag:`poolstruct.extend` a precalculated ``extendBy`` field (see
:mps:ref:`.init.extend`). This value is used to determine how large a
segment should be requested from the arena for the reference part
segment when the pool needs to grow (see :mps:ref:`.alloc.grow.size`).
:mps:tag:`poolstruct.extend.justify` Calculating a reasonable value for this
once and remembering it simplifies the allocation (:mps:ref:`.alloc.grow`).
:mps:tag:`poolstruct.init` poolstructs are initialized once for each pool
instance by :c:func:`MRGInit()` (:mps:ref:`.init`). The initial state has all the
rings initialized to singleton rings, and the ``extendBy`` field
initialized to some value (see :mps:ref:`.init.extend`).
:mps:tag:`mrgseg` The pool defines two segment subclasses:
:c:type:`MRGRefSegClass` and :c:type:`MRGLinkSegClass`. Segments of the former
class will be used to store the ref parts of guardians, segments of
the latter will be used to store the link parts of guardians (see
:mps:ref:`.guardian.two-part`). Segments are always allocated in pairs, with
one of each class, by the function :c:func:`MRGSegPairCreate()`. Each
segment contains a link to its pair.
:mps:tag:`mrgseg.ref` :c:type:`MRGRefSegClass` is a subclass of :c:type:`GCSegClass`.
Instances are of type ``MRGRefSeg``, and contain:
- :mps:tag:`mrgseg.ref.mrgring` a field for the ring of ref part segments in
the pool.
- :mps:tag:`mrgseg.ref.linkseg` a pointer to the paired link segment.
- :mps:tag:`mrgseg.ref.grey` a set describing the greyness of the segment for each trace.
:mps:tag:`mrgseg.ref.init` A segment is created and initialized once every
time the pool is grown (:mps:ref:`.alloc.grow`). The initial state has the
segment ring node initialized and attached to the pool's segment ring,
the linkseg field points to the relevant link segment, the grey field
is initialized such that the segment is not grey for all traces.
:mps:tag:`mrgseg.link` :c:type:`MRGLinkSegClass` is a subclass of :c:type:`SegClass`.
Instances are of type ``MRGLinkSeg``, and contain:
- :mps:tag:`mrgseg.link.refseg` a pointer to the paired ref segment. This
may be :c:macro:`NULL` during initialization, while the pairing is being
established.
- :mps:tag:`mrgseg.link.init` The initial state has the ``linkseg`` field
pointing to the relevant ref segment.
Functions
---------
.. c:function:: Bool MRGCheck(MRG mrg)
:mps:tag:`check` Check the signatures, the class, and each field of the
:c:type:`MRGStruct`. Each field is checked as being appropriate for its
type.
:mps:tag:`check.justify` There are no non-trivial invariants that can
be easily checked.
.. c:function:: Res MRGRegister(Pool pool, Ref ref)
:mps:tag:`alloc` Add a guardian for ``ref``.
:mps:tag:`alloc.grow` If the free list is empty then two new segments are
allocated and the free list filled up from them (note that the
reference fields of the new guardians will need to be overwritten with
:c:macro:`NULL`, see :mps:ref:`.free.overwrite`)
:mps:tag:`alloc.grow.size` The size of the reference part segment will be
the pool's ``extendBy`` (:mps:ref:`.poolstruct.extend`) value. The link part
segment will be whatever size is necessary to accommodate *N* link
parts, where *N* is the number of reference parts that fit in the
reference part segment.
:mps:tag:`alloc.error` If any of the requests for more resource (there are
two; one for each of two segments) fail then the successful requests
will be retracted and the result code from the failing request will be
returned.
:mps:tag:`alloc.pop` :c:func:`MRGRegister()` pops a ring node off the free list,
and add it to the entry list.
.. c:function:: Res MRGDeregister(Pool pool, Ref obj)
:mps:tag:`free` Remove the guardian from the message queue and add it to the
free list.
:mps:tag:`free.push` The guardian will simply be added to the front of the
free list (that is, no keeping the free list in address order or
anything like that).
:mps:tag:`free.inadequate` No attempt will be made to return unused free
segments to the arena (although see
analysis.mps.poolmrg.improve.free.* for suggestions).
:mps:tag:`free.overwrite` :c:func:`MRGDeregister()` also writes over the reference
with :c:macro:`NULL`. :mps:tag:`free.overwrite.justify` This is so that when the
segment is subsequently scanned (:mps:ref:`.scan.free`), the reference that
used to be in the object is not accidentally fixed.
.. c:function:: Res MRGInit(Pool pool, ArgList args)
:mps:tag:`init` Initializes the entry list, the free ring, the ref ring, and
the ``extendBy`` field.
:mps:tag:`init.extend` The ``extendBy`` field is initialized to one
:c:func:`ArenaAlign()` (usually a page).
:mps:tag:`init.extend.justify` This is adequate as the pool is not expected
to grow very quickly.
.. c:function:: void MRGFinish(Pool pool)
:mps:tag:`finish` Iterate over all the segments, returning all the segments
to the arena.
.. c:function:: Res MRGScan(Bool *totalReturn, ScanState ss, Pool pool, Seg seg)
:mps:tag:`scan` :c:func:`MRGScan()` scans a segment.
:mps:tag:`scan.trivial` Scan will do nothing (that is, return immediately)
if the tracing rank is anything other than final.
.. note::
This optimization is missing. impl.c.trace.scan.conservative is
not a problem because there are no faults on these segs, because
there are no references into them. But that's why :c:func:`TraceScan()`
can't do it. Pekka P. Pirinen, 1997-09-19.
:mps:tag:`scan.trivial.justify` If the rank is lower than final then
scanning is detrimental, it will only delay finalization. If the rank
is higher than final there is nothing to do, the pool only contains
final references.
:mps:tag:`scan.guardians` :c:func:`MRGScan()` will iterate over all guardians in
the segment. Every guardian's reference will be fixed (:mps:tag:`scan.free`
note that guardians that are on the free list have :c:macro:`NULL` in their
reference part).
:mps:tag:`scan.wasold` If the object referred to had not been fixed
previously (that is, was unmarked) then the object is not referenced
by a reference of a lower rank (than ``RankFINAL``) and hence is
finalizable.
:mps:tag:`scan.finalize` The guardian will be finalized. This entails moving
the guardian from state Prefinal to Final; it is removed from the
entry list and initialized as a message and posted on the arena's
message queue.
:mps:tag:`scan.finalize.idempotent` In fact this will only happen if the
guardian has not already been finalized (which is determined by
examining the state of the guardian).
:mps:tag:`scan.unordered` Because scanning occurs a segment at a time, the
order in which objects are finalized is "random" (it cannot be
predicted by considering only the references between objects
registered for finalization). See
analysis.mps.poolmrg.improve.semantics for how this can be improved.
:mps:tag:`scan.unordered.justify` Unordered finalization is all that is
required.
See analysis.mps.poolmrg.improve.scan.nomove for a suggested
improvement that avoids redundant unlinking and relinking.
.. c:function:: Res MRGDescribe(Pool pool, mps_lib_FILE *stream)
:mps:tag:`describe` Describes an MRG pool. Iterates along each of the entry
and exit lists and prints the guardians in each. The location of the
guardian and the value of the reference in it will be printed out.
:mps:tag:`functions.unused` All of these will be unused: :c:func:`BufferInit()`,
:c:func:`BufferFill()`, :c:func:`BufferEmpty()`, :c:func:`BufferFinish()`,
:c:func:`TraceBegin()`, :c:func:`TraceCondemn()`, :c:func:`PoolFix()`, :c:func:`PoolReclaim()`, :c:func:`TraceEnd()`.
:mps:tag:`functions.trivial` The Grey method of the pool class will be
:c:func:`PoolTrivGrey()`, this pool has no further bookkeeping to perform
for grey segments.
Transgressions
--------------
:mps:tag:`trans.no-finish` The MRG pool does not trouble itself to tidy up
its internal rings properly when being destroyed.
:mps:tag:`trans.free-seg` No attempt is made to release free segments to the
arena. A suggested strategy for this is as follows:
- Add a count of free guardians to each segment, and maintain it in
appropriate places.
- Add a free segment ring to the pool.
- In :c:func:`MRGRefSegScan()`, if the segment is entirely free, don't scan
it, but instead detach its links from the free ring, and move the
segment to the free segment ring.
- At some appropriate point, such as the end of :c:func:`MRGAlloc()`,
destroy free segments.
- In :c:func:`MRGAlloc()`, if there are no free guardians, check the free
segment ring before creating a new pair of segments. Note that this
algorithm would give some slight measure of segment hysteresis. It
is not the place of the pool to support general segment hysteresis.
Future
------
:mps:tag:`future.array` In future, for speed or simplicity, this pool could
be rewritten to use an array. See mail.gavinm.1997-09-04.13-08(0).
Tests
-----
.. note::
This section is utterly out of date. Pekka P. Pirinen, 1997-09-19.
:mps:tag:`test` The test impl.c.finalcv is similar to the weakness test (see
design.mps.weakness, impl.c.weakcv).
Functionality
.............
This is the functionality to be tested:
- :mps:tag:`fun.alloc` Can allocate objects.
- :mps:tag:`fun.free` Can free objects that were allocated.
- :mps:tag:`prot.write` Can write a reference into an allocated object.
- :mps:tag:`prot.read` Can read the reference from an allocated object.
- :mps:tag:`promise.faithful` A reference stored in an allocated object will
continue to refer to the same object.
- :mps:tag:`promise.live` A reference stored in an allocated object will
preserve the object referred to.
- :mps:tag:`promise.unreachable` Any objects referred to in finalization
messages are not (at the time of reading the message) reachable via
a chain of ambiguous or exact references. (we will not be able to
test this at first as there is no messaging interface)
- :mps:tag:`promise.try` The pool will make a "good faith" effort to
finalize objects that are not reachable via a chain of ambiguous or
exact references.
Attributes
..........
The following attributes will be tested:
- :mps:tag:`attr.none` There are no attribute requirements.
Implementation
..............
The test will simply allocate a number of objects in the AMC pool and
finalize each one, throwing away the reference to the objects. Churn.
:mps:tag:`test.mpm` The test will use the MPM interface (impl.h.mpm).
:mps:tag:`test.mpm.justify` This is because it is not intended to provide an
MPS interface to this pool directly, and the MPS interface to
finalization has not been written yet (impl.h.mps).
:mps:tag:`test.mpm.change` Later on it may use the MPS interface, in which
case, where the following text refers to allocating objects in the MRG
pool it will need adjusting.
:mps:tag:`test.two-pools` The test will use two pools, an AMC pool, and an
MRG pool.
:mps:tag:`test.alloc` A number of objects will be allocated in the MRG pool.
:mps:tag:`test.free` They will then be freed. This will test :mps:ref:`.fun.alloc`
and :mps:ref:`.fun.free`, although not very much.
:mps:tag:`test.rw.a` An object, "A", will be allocated in the AMC pool, a
reference to it will be kept in a root.
:mps:tag:`test.rw.alloc` A number of objects will be allocated in the MRG
pool.
:mps:tag:`test.rw.write` A reference to "A" will be written into each
object.
:mps:tag:`test.rw.read` The reference in each object will be read and
checked to see if it refers to "A".
:mps:tag:`test.rw.free` All the objects will be freed.
:mps:tag:`test.rw.drop` The reference to "A" will be dropped. This will test
:mps:ref:`.prot.write` and :mps:ref:`.prot.read`.
:mps:tag:`test.promise.fl.alloc` A number of objects will be allocated in
the AMC pool.
:mps:tag:`test.promise.fl.tag` Each object will be tagged uniquely.
:mps:tag:`test.promise.fl.refer` a reference to it will be stored in an
object allocated in the MRG pool.
:mps:tag:`test.promise.fl.churn` A large amount of garbage will be allocated
in the AMC pool. Regularly, whilst this garbage is being allocated, a
check will be performed that all the objects allocated in the MRG pool
refer to valid objects and that they still refer to the same objects.
All objects from the MRG pool will then be freed (thus dropping all
references to the AMC objects). This will test :mps:ref:`.promise.faithful`
and :mps:ref:`.promise.live`.
:mps:tag:`test.promise.ut.not` The following part of the test has not
implemented. This is because the messaging system has not yet been
implemented.
:mps:tag:`test.promise.ut.alloc` A number of objects will be allocated in
the AMC pool.
:mps:tag:`test.promise.ut.refer` Each object will be referred to by a root
and also referred to by an object allocated in the MRG pool.
:mps:tag:`test.promise.ut.drop` References to a random selection of the
objects from the AMC pool will be deleted from the root.
:mps:tag:`test.promise.ut.churn` A large amount of garbage will be allocated
in the AMC pool.
:mps:tag:`test.promise.ut.message` The message interface will be used to
receive finalization messages.
:mps:tag:`test.promise.ut.final.check` For each finalization message
received it will check that the object referenced in the message is
not referred to in the root.
:mps:tag:`test.promise.ut.nofinal.check` After some amount of garbage has
been allocated it will check to see if any objects are not in the root
and haven't been finalized. This will test :mps:ref:`.promise.unreachable` and
:mps:ref:`.promise.try`.
Notes
-----
:mps:tag:`access.inadequate` :c:func:`PoolAccess()` will scan segments at
`RankEXACT``. Really it should be scanned at whatever the minimum rank
of all grey segments is (the trace rank phase), however there is no
way to find this out. As a consequence we will sometimes scan pages at
``RankEXACT`` when the pages could have been scanned at ``RankFINAL``.
This means that finalization of some objects may sometimes get
delayed.

Some files were not shown because too many files have changed in this diff Show more