diff options
Diffstat (limited to 'pytest/protocol.py')
| -rw-r--r-- | pytest/protocol.py | 1998 |
1 files changed, 1998 insertions, 0 deletions
diff --git a/pytest/protocol.py b/pytest/protocol.py new file mode 100644 index 000000000000..6b0cd9ad0327 --- /dev/null +++ b/pytest/protocol.py @@ -0,0 +1,1998 @@ +#! /usr/bin/env python + +""" +Protocol definitions for python based lib9p server/client. + +The sub-namespace td has type definitions (qid, stat) and values +that are "#define" constants in C code (e.g., DMDIR, QTFILE, etc). +This also contains the byte values for protocol codes like Tversion, +Rversion, Rerror, and so on. + + >>> td.Tversion + 100 + >>> td.Rlerror + 7 + +The qid and stat types are PFOD classes and generate instances that +are a cross between namedtuple and OrderedDictionary (see pfod.py +for details): + + >>> td.qid(type=td.QTFILE, path=2, version=1) + qid(type=0, version=1, path=2) + +The td.stat() type output is pretty long, since it has all the +dotu-specific members (used only when packing for dotu/dotl and +set only when unpacking those), so here's just one field: + + >>> td.stat(*(15 * [0])).mode + 0 + >>> import pprint; pprint.pprint(td.stat()._fields) + ('type', + 'dev', + 'qid', + 'mode', + 'atime', + 'mtime', + 'length', + 'name', + 'uid', + 'gid', + 'muid', + 'extension', + 'n_uid', + 'n_gid', + 'n_muid') + +Stat objects sent across the protocol must first be encoded into +wirestat objects, which are basically size-counted pre-sequenced +stat objects. The pre-sequencing uses: + + >>> td.stat_seq + Sequencer('stat') + +For parsing bytes returned in a Tread on a directory, td.wirestat_seq +is the sequencer. However, most users should rely on the packers and +unpackers in each protocol (see {pack,unpack}_wirestat below). + + >>> td.wirestat_seq + Sequencer('wirestat') + +There is a dictionary fcall_to_name that maps from byte value +to protocol code. Names map to themselves as well: + + >>> fcall_names[101] + 'Rversion' + >>> fcall_names['Tversion'] + 'Tversion' + +The sub-namespace rrd has request (Tversion, Topen, etc) and +response (Rversion, Ropen, etc) data definitions. Each of these +is a PFOD class: + + >>> rrd.Tversion(1000, 'hello', tag=0) + Tversion(tag=0, msize=1000, version='hello') + +The function p9_version() looks up the instance of each supported +protocol, or raises a KeyError when given an invalid protocol. +The names may be spelled in any mixture of cases. + +The names plain, dotu, and dotl are predefined as the three +supported protocols: + + >>> p9_version('invalid') + Traceback (most recent call last): + ... + KeyError: 'invalid' + >>> p9_version('9p2000') == plain + True + >>> p9_version('9P2000') == plain + True + >>> p9_version('9P2000.u') == dotu + True + >>> p9_version('9p2000.L') == dotl + True + +Protocol instances have a pack() method that encodes a set of +arguments into a packet. To know what to encode, pack() must +receive an fcall value and a dictionary containing argument +values, or something equivalent. The required argument values +depend on the fcall. For instance, a Tversion fcall needs three +arguments: the version name, the tag, and the msize (these of +course are the pre-filled fields in a Tversion PFOD instance). + + >>> args = {'version': '!', 'tag': 1, 'msize': 1000} + >>> pkt = dotu.pack(fcall='Tversion', args=args) + >>> len(pkt) + 14 + +The length of string '!' is 1, and the packet (or wire) format of +a Tversion request is: + + size[4] fcall[1] tag[2] msize[4] version[s] + +which corresponds to a struct's IBHIH (for the fixed size parts) +followed by 1 B (for the string). The overall packet is 14 bytes +long, so we have size=9, fcall=100, tag=1, msize=1000, and the +version string is length=1, value=33 (ord('!')). + + >>> import struct + >>> struct.unpack('<IBHIHB', pkt) + (14, 100, 1, 1000, 1, 33) + +Of course, this packed a completely bogus "version" string, but +that's what we told it to do. Protocol instances remember their +version, so we can get it right by omitting the version from the +arguments: + + >>> dotu.version + '9P2000.u' + >>> args = {'tag': 99, 'msize': 1000} + >>> pkt = dotu.pack(fcall='Tversion', args=args) + >>> len(pkt) + 21 + +The fcall can be supplied numerically: + + >>> pkt2 = dotu.pack(fcall=td.Tversion, args=args) + >>> pkt == pkt2 + True + +Instead of providing an fcall you can provide an instance of +the appropriate PFOD. In this case pack() finds the type from +the PFOD instance. As usual, the version parameter is filled in +for you: + + >>> pkt2 = dotu.pack(rrd.Tversion(tag=99, msize=1000)) + >>> pkt == pkt2 + True + +Note that it's up to you to check the other end's version and +switch to a "lower" protocol as needed. Each instance does provide +a downgrade_to() method that gets you a possibly-downgraded instance. +This will fail if you are actually trying to upgrade, and also if +you provide a bogus version: + + >>> dotu.downgrade_to('9P2000.L') + Traceback (most recent call last): + ... + KeyError: '9P2000.L' + >>> dotu.downgrade_to('we never heard of this protocol') + Traceback (most recent call last): + ... + KeyError: 'we never heard of this protocol' + +Hence you might use: + + try: + proto = protocol.dotl.downgrade(vstr) + except KeyError: + pkt = protocol.plain.pack(fcall='Rerror', + args={'tag': tag, 'errstr': 'unknown protocol version ' + '{0!r}'.format(vstr)}) + else: + pkt = proto.pack(fcall='Rversion', args={'tag': tag, 'msize': msize}) + +When using a PFOD instance, it is slightly more efficient to use +pack_from(): + + try: + proto = protocol.dotl.downgrade(vstr) + reply = protocol.rrd.Rversion(tag=tag, msize=msize) + except KeyError: + proto = protocol.plain + reply = protocol.rrd.Rerror(tag=tag, + errstr='unknown protocol version {0!r}'.format(vstr)) + pkt = proto.pack_from(reply) + +does the equivalent of the try/except/else variant. Note that +the protocol.rrd.Rversion() instance has version=None. Like +proto.pack, the pack_from will detect this "missing" value and +fill it in. + +Because errors vary (one should use Rlerror for dotl and Rerror +for dotu and plain), and it's convenient to use an Exception +instance for an error, all protocols provide .error(). This +builds the appropriate kind of error response, extracting and +converting errno's and error messages as appropriate. + +If <err> is an instance of Exception, err.errno provides the errnum +or ecode value (if used, for dotu and dotl) and err.strerror as the +errstr value (if used, for plain 9p2000). Otherwise err should be +an integer, and we'll use os.strerror() to get a message. + +When using plain 9P2000 this sends error *messages*: + + >>> import errno, os + >>> utf8 = os.strerror(errno.ENOENT).encode('utf-8') + >>> pkt = None + >>> try: + ... os.open('presumably this file does not exist here', 0) + ... except OSError as err: + ... pkt = plain.error(1, err) + ... + >>> pkt[-len(utf8):] == utf8 + True + >>> pkt2 = plain.error(1, errno.ENOENT) + >>> pkt == pkt2 + True + +When using 9P2000.u it sends the error code as well, and when +using 9P2000.L it sends only the error code (and more error +codes can pass through): + + >>> len(pkt) + 34 + >>> len(dotu.error(1, errno.ENOENT)) + 38 + >>> len(dotl.error(1, errno.ENOENT)) + 11 + +For even more convenience (and another slight speed hack), the +protocol has member functions for each valid pfod, which +effectively do a pack_from of a pfod built from the arguments. In +the above example this is not very useful (because we want two +different replies), but for Rlink, for instance, which has only +a tag, a server might implement Tlink() as: + + def do_Tlink(proto, data): # data will be a protocol.rrd.Tlink(...) + tag = data.tag + dfid = data.dfid + fid = data.fid + name = data.name + ... some code to set up for doing the link link ... + try: + os.link(path1, path2) + except OSError as err: + return proto.error(tag, err) + else: + return proto.Rlink(tag) + + >>> pkt = dotl.Rlink(12345) + >>> struct.unpack('<IBH', pkt) + (7, 71, 12345) + +Similarly, a client can build a Tversion packet quite trivially: + + >>> vpkt = dotl.Tversion(tag=0, msize=12345) + +To see that this is a valid version packet, let's unpack its bytes. +The overall length is 21 bytes: 4 bytes of size, 1 byte of code 100 +for Tversion, 2 bytes of tag, 4 bytes of msize, 2 bytes of string +length, and 8 bytes of string '9P2000.L'. + + >>> tup = struct.unpack('<IBHIH8B', vpkt) + >>> tup[0:5] + (21, 100, 0, 12345, 8) + >>> ''.join(chr(i) for i in tup[5:]) + '9P2000.L' + +Of course, since you can *pack*, you can also *unpack*. It's +possible that the incoming packet is malformed. If so, this +raises various errors (see below). + +Unpack is actually a two step process: first we unpack a header +(where the size is already removed and is implied by len(data)), +then we unpack the data within the packet. You can invoke the +first step separately. Furthermore, there's a noerror argument +that leaves some fields set to None or empty strings, if the +packet is too short. (Note that we need a hack for py2k vs py3k +strings here, for doctests. Also, encoding 12345 into a byte +string produces '90', by ASCII luck!) + + >>> pkt = pkt[4:] # strip generated size + >>> import sys + >>> py3k = sys.version_info[0] >= 3 + >>> b2s = lambda x: x.decode('utf-8') if py3k else x + >>> d = plain.unpack_header(pkt[0:1], noerror=True) + >>> d.data = b2s(d.data) + >>> d + Header(size=5, dsize=0, fcall=71, data='') + >>> d = plain.unpack_header(pkt[0:2], noerror=True) + >>> d.data = b2s(d.data) + >>> d + Header(size=6, dsize=1, fcall=71, data='9') + +Without noerror=True a short packet raises a SequenceError: + + >>> plain.unpack_header(pkt[0:0]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SequenceError: out of data while unpacking 'fcall' + +Of course, a normal packet decodes fine: + + >>> d = plain.unpack_header(pkt) + >>> d.data = b2s(d.data) + >>> d + Header(size=7, dsize=2, fcall=71, data='90') + +but one that is too *long* potentially raises a SequencError. +(This is impossible for a header, though, since the size and +data size are both implied: either there is an fcall code, and +the rest of the bytes are "data", or there isn't and the packet +is too short. So we can only demonstrate this for regular +unpack; see below.) + +Note that all along, this has been decoding Rlink (fcall=71), +which is not valid for plain 9P2000 protocol. It's up to the +caller to check: + + >>> plain.supports(71) + False + + >>> plain.unpack(pkt) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SequenceError: invalid fcall 'Rlink' for 9P2000 + >>> dotl.unpack(pkt) + Rlink(tag=12345) + +However, the unpack() method DOES check that the fcall type is +valid, even if you supply noerror=True. This is because we can +only really decode the header, not the data, if the fcall is +invalid: + + >>> plain.unpack(pkt, noerror=True) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SequenceError: invalid fcall 'Rlink' for 9P2000 + +The same applies to much-too-short packets even if noerror is set. +Specifically, if the (post-"size") header shortens down to the empty +string, the fcall will be None: + + >>> dotl.unpack(b'', noerror=True) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SequenceError: invalid fcall None for 9P2000.L + +If there is at least a full header, though, noerror will do the obvious: + + >>> dotl.unpack(pkt[0:1], noerror=True) + Rlink(tag=None) + >>> dotl.unpack(pkt[0:2], noerror=True) + Rlink(tag=None) + +If the packet is too long, noerror suppresses the SequenceError: + + >>> dotl.unpack(pkt + b'x') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SequenceError: 1 byte(s) unconsumed + >>> dotl.unpack(pkt + b'x', noerror=True) + Rlink(tag=12345) + +To pack a stat object when producing data for reading a directory, +use pack_wirestat. This puts a size in front of the packed stat +data (they're represented this way in read()-of-directory data, +but not elsewhere). + +To unpack the result of a Tstat or a read() on a directory, use +unpack_wirestat. The stat values are variable length so this +works with offsets. If the packet is truncated, you'll get a +SequenceError, but just as for header unpacking, you can use +noerror to suppress this. + +(First, we'll need to build some valid packet data.) + + >>> statobj = td.stat(type=0,dev=0,qid=td.qid(0,0,0),mode=0, + ... atime=0,mtime=0,length=0,name=b'foo',uid=b'0',gid=b'0',muid=b'0') + >>> data = plain.pack_wirestat(statobj) + >>> len(data) + 55 + +Now we can unpack it: + + >>> newobj, offset = plain.unpack_wirestat(data, 0) + >>> newobj == statobj + True + >>> offset + 55 + +Since the packed data do not include the dotu extensions, we get +a SequenceError if we try to unpack with dotu or dotl: + + >>> dotu.unpack_wirestat(data, 0) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SequenceError: out of data while unpacking 'extension' + +When using noerror, the returned new offset will be greater +than the length of the packet, after a failed unpack, and some +elements may be None: + + >>> newobj, offset = plain.unpack_wirestat(data[0:10], 0, noerror=True) + >>> offset + 55 + >>> newobj.length is None + True + +Similarly, use unpack_dirent to unpack the result of a dot-L +readdir(), using offsets. (Build them with pack_dirent.) + + >>> dirent = td.dirent(qid=td.qid(1,2,3),offset=0, + ... type=td.DT_REG,name=b'foo') + >>> pkt = dotl.pack_dirent(dirent) + >>> len(pkt) + 27 + +and then: + + >>> newde, offset = dotl.unpack_dirent(pkt, 0) + >>> newde == dirent + True + >>> offset + 27 + +""" + +from __future__ import print_function + +import collections +import os +import re +import sys + +import p9err +import pfod +import sequencer + +SequenceError = sequencer.SequenceError + +fcall_names = {} + +# begin ??? +# to interfere with (eg) the size part of the packet: +# pkt = proto.pack(fcall=protocol.td.Tversion, +# size=123, # wrong +# args={ 'tag': 1, msize: 1000, version: '9p2000.u' }) +# a standard Twrite: +# pkt = proto.pack(fcall=protocol.td.Twrite, +# args={ 'tag': 1, 'fid': 2, 'offset': 0, 'data': b'rawdata' }) +# or: +# pkt = proto.pack(fcall=protocol.td.Twrite, +# data=proto.Twrite(tag=1, fid=2, offset=0, data=b'rawdata' }) +# a broken Twrite: +# pkt = proto.pack(fcall=protocol.td.Twrite, +# args={ 'tag': 1, 'fid': 2, 'offset': 0, 'count': 99, +# 'data': b'rawdata' }) -- XXX won't work (yet?) +# +# build a QID: (td => typedefs and defines) +# qid = protocol.td.qid(type=protocol.td.QTFILE, version=1, path=2) +# build the Twrite data as a data structure: +# wrdata = protocol.td.Twrite(tag=1, fid=2, offset=0, data=b'rawdata') +# +# turn incoming byte stream data into a Header and remaining data: +# foo = proto.pack(data) + +class _PackInfo(object): + """ + Essentially just a Sequencer, except that we remember + if there are any :auto annotations on any of the coders, + and we check for coders that are string coders ('data[size]'). + + This could in theory be a recursive check, but in practice + all the automatics are at the top level, and we have no mechanism + to pass down inner automatics. + """ + def __init__(self, seq): + self.seq = seq + self.autos = None + for pair in seq: # (cond, code) pair + sub = pair[1] + if sub.aux is None: + continue + assert sub.aux == 'auto' or sub.aux == 'len' + if self.autos is None: + self.autos = [] + self.autos.append(pair) + + def __repr__(self): + return '{0}({1!r})'.format(self.__class__.__name__, self.seq) + + def pack(self, auto_vars, conditions, data, rodata): + """ + Pack data. Insert automatic and/or counted variables + automatically, if they are not already set in the data. + + If rodata ("read-only data") is True we make sure not + to modify the caller's data. Since data is a PFOD rather + than a normal ordered dictionary, we use _copy(). + """ + if self.autos: + for cond, sub in self.autos: + # False conditionals don't need to be filled-in. + if cond is not None and not conditions[cond]: + continue + if sub.aux == 'auto': + # Automatic variable, e.g., version. The + # sub-coder's name ('version') is the test item. + if data.get(sub.name) is None: + if rodata: + data = data._copy() + rodata = False + data[sub.name] = auto_vars[sub.name] + else: + # Automatic length, e.g., data[count]. The + # sub-coders's repeat item ('count') is the + # test item. Of course, it's possible that + # the counted item is missing as well. If so + # we just leave both None and take the + # encoding error. + assert sub.aux == 'len' + if data.get(sub.repeat) is not None: + continue + item = data.get(sub.name) + if item is not None: + if rodata: + data = data._copy() + rodata = False + data[sub.repeat] = len(item) + return self.seq.pack(data, conditions) + +class _P9Proto(object): + def __init__(self, auto_vars, conditions, p9_data, pfods, index): + self.auto_vars = auto_vars # currently, just version + self.conditions = conditions # '.u' + self.pfods = pfods # dictionary, maps pfod to packinfo + self.index = index # for comparison: plain < dotu < dotl + + self.use_rlerror = rrd.Rlerror in pfods + + for dtype in pfods: + name = dtype.__name__ + # For each Txxx/Rxxx, define a self.<name>() to + # call self.pack_from(). + # + # The packinfo is from _Packinfo(seq); the fcall and + # seq come from p9_data.protocol[<name>]. + proto_tuple = p9_data.protocol[name] + assert dtype == proto_tuple[0] + packinfo = pfods[dtype] + # in theory we can do this with no names using nested + # lambdas, but that's just too confusing, so let's + # do it with nested functions instead. + def builder(constructor=dtype, packinfo=packinfo): + "return function that calls _pack_from with built PFOD" + def invoker(self, *args, **kwargs): + "build PFOD and call _pack_from" + return self._pack_from(constructor(*args, **kwargs), + rodata=False, caller=None, + packinfo=packinfo) + return invoker + func = builder() + func.__name__ = name + func.__doc__ = 'pack from {0}'.format(name) + setattr(self.__class__, name, func) + + def __repr__(self): + return '{0}({1!r})'.format(self.__class__.__name__, self.version) + + def __str__(self): + return self.version + + # define rich-comparison operators, so we can, e.g., test vers > plain + def __lt__(self, other): + return self.index < other.index + def __le__(self, other): + return self.index <= other.index + def __eq__(self, other): + return self.index == other.index + def __ne__(self, other): + return self.index != other.index + def __gt__(self, other): + return self.index > other.index + def __ge__(self, other): + return self.index >= other.index + + def downgrade_to(self, other_name): + """ + Downgrade from this protocol to a not-greater one. + + Raises KeyError if other_name is not a valid protocol, + or this is not a downgrade (with setting back to self + considered a valid "downgrade", i.e., we're doing subseteq + rather than subset). + """ + if not isinstance(other_name, str) and isinstance(other_name, bytes): + other_name = other_name.decode('utf-8', 'surrogateescape') + other = p9_version(other_name) + if other > self: + raise KeyError(other_name) + return other + + def error(self, tag, err): + "produce Rerror or Rlerror, whichever is appropriate" + if isinstance(err, Exception): + errnum = err.errno + errmsg = err.strerror + else: + errnum = err + errmsg = os.strerror(errnum) + if self.use_rlerror: + return self.Rlerror(tag=tag, ecode=p9err.to_dotl(errnum)) + return self.Rerror(tag=tag, errstr=errmsg, + errnum=p9err.to_dotu(errnum)) + + def pack(self, *args, **kwargs): + "pack up a pfod or fcall-and-arguments" + fcall = kwargs.pop('fcall', None) + if fcall is None: + # Called without fcall=... + # This requires that args have one argument that + # is the PFOD; kwargs should be empty (but we'll take + # data=pfod as well). The size is implied, and + # fcall comes from the pfod. + data = kwargs.pop('data', None) + if data is None: + if len(args) != 1: + raise TypeError('pack() with no fcall requires 1 argument') + data = args[0] + if len(kwargs): + raise TypeError('pack() got an unexpected keyword argument ' + '{0}'.format(kwargs.popitem()[0])) + return self._pack_from(data, True, 'pack', None) + + # Called as pack(fcall=whatever, data={...}). + # The data argument must be a dictionary since we're going to + # apply ** to it in the call to build the PFOD. Note that + # it could already be a PFOD, which is OK, but we're going to + # copy it to a new one regardless (callers that have a PFOD + # should use pack_from instead). + if len(args): + raise TypeError('pack() got unexpected arguments ' + '{0!r}'.format(args)) + data = kwargs.pop('args', None) + if len(kwargs): + raise TypeError('pack() got an unexpected keyword argument ' + '{0}'.format(kwargs.popitem()[0])) + if not isinstance(data, dict): + raise TypeError('pack() with fcall and data ' + 'requires data to be a dictionary') + try: + name = fcall_names[fcall] + except KeyError: + raise TypeError('pack(): {0} is not a valid ' + 'fcall value'.format(fcall)) + cls = getattr(rrd, name) + data = cls(**data) + return self._pack_from(data, False, 'pack', None) + + def pack_from(self, data): + "pack from pfod data, using its type to determine fcall" + return self._pack_from(data, True, 'pack_from', None) + + def _pack_from(self, data, rodata, caller, packinfo): + """ + Internal pack(): called from both invokers (self.Tversion, + self.Rwalk, etc.) and from pack and pack_from methods. + "caller" says which. If rodata is True we're not supposed to + modify the incoming data, as it may belong to someone + else. Some calls to pack() build a PFOD and hence pass in + False. + + The predefined invokers pass in a preconstructed PFOD, + *and* set rodata=False, *and* provide a packinfo, so that + we never have to copy, nor look up the packinfo. + """ + if caller is not None: + assert caller in ('pack', 'pack_from') and packinfo is None + # Indirect call from pack_from(), or from pack() after + # pack() built a PFOD. We make sure this kind of PFOD + # is allowed for this protocol. + packinfo = self.pfods.get(data.__class__, None) + if packinfo is None: + raise TypeError('{0}({1!r}): invalid ' + 'input'.format(caller, data)) + + # Pack the data + pkt = packinfo.pack(self.auto_vars, self.conditions, data, rodata) + + fcall = data.__class__.__name__ + fcall_code = getattr(td, fcall) + + # That's the inner data; now we must add the header, + # with fcall (translated back to byte code value) and + # outer data. The size is implied by len(pkt). There + # are no other auto variables, and no conditions. + # + # NB: the size includes the size of the header itself + # and the fcall code byte, plus the size of the data. + data = _9p_data.header_pfod(size=4 + 1 + len(pkt), dsize=len(pkt), + fcall=fcall_code, data=pkt) + empty = None # logically should be {}, but not actually used below + pkt = _9p_data.header_pack_seq.pack(data, empty) + return pkt + + @staticmethod + def unpack_header(bstring, noerror=False): + """ + Unpack header. + + We know that our caller has already stripped off the + overall size field (4 bytes), leaving us with the fcall + (1 byte) and data (len(bstring)-1 bytes). If len(bstring) + is 0, this is an invalid header: set dsize to 0 and let + fcall become None, if noerror is set. + """ + vdict = _9p_data.header_pfod() + vdict['size'] = len(bstring) + 4 + vdict['dsize'] = max(0, len(bstring) - 1) + _9p_data.header_unpack_seq.unpack(vdict, None, bstring, noerror) + return vdict + + def unpack(self, bstring, noerror=False): + "produce filled PFOD from fcall in packet" + vdict = self.unpack_header(bstring, noerror) + # NB: vdict['dsize'] is used internally during unpack, to + # find out how many bytes to copy to vdict['data'], but by + # the time unpack is done, we no longer need it. + # + # size = vdict['size'] + # dsize = vdict['dsize'] + fcall = vdict['fcall'] + data = vdict['data'] + # Note: it's possible for size and/or fcall to be None, + # when noerror is true. However, if we support fcall, then + # clearly fcall is not None; and since fcall follows size, + # we can always proceed if we support fcall. + if self.supports(fcall): + fcall = fcall_names[fcall] + cls = getattr(rrd, fcall) + seq = self.pfods[cls].seq + elif fcall == td.Rlerror: + # As a special case for diod, we accept Rlerror even + # if it's not formally part of the protocol. + cls = rrd.Rlerror + seq = dotl.pfods[rrd.Rlerror].seq + else: + fcall = fcall_names.get(fcall, fcall) + raise SequenceError('invalid fcall {0!r} for ' + '{1}'.format(fcall, self)) + vdict = cls() + seq.unpack(vdict, self.conditions, data, noerror) + return vdict + + def pack_wirestat(self, statobj): + """ + Pack a stat object to appear as data returned by read() + on a directory. Essentially, we prefix the data with a size. + """ + data = td.stat_seq.pack(statobj, self.conditions) + return td.wirestat_seq.pack({'size': len(data), 'data': data}, {}) + + def unpack_wirestat(self, bstring, offset, noerror=False): + """ + Produce the next td.stat object from byte-string, + returning it and new offset. + """ + statobj = td.stat() + d = { 'size': None } + newoff = td.wirestat_seq.unpack_from(d, self.conditions, bstring, + offset, noerror) + size = d['size'] + if size is None: # implies noerror; newoff==offset+2 + return statobj, newoff + # We now have size and data. If noerror, data might be + # too short, in which case we'll unpack a partial statobj. + # Or (with or without noeror), data might be too long, so + # that while len(data) == size, not all the data get used. + # That may be allowed by the protocol: it's not clear. + data = d['data'] + used = td.stat_seq.unpack_from(statobj, self.conditions, data, + 0, noerror) + # if size != used ... then what? + return statobj, newoff + + def pack_dirent(self, dirent): + """ + Dirents (dot-L only) are easy to pack, but we provide + this function for symmetry. (Should we raise an error + if called on plain or dotu?) + """ + return td.dirent_seq.pack(dirent, self.conditions) + + def unpack_dirent(self, bstring, offset, noerror=False): + """ + Produces the next td.dirent object from byte-string, + returning it and new offset. + """ + deobj = td.dirent() + offset = td.dirent_seq.unpack_from(deobj, self.conditions, bstring, + offset, noerror) + return deobj, offset + + def supports(self, fcall): + """ + Return True if and only if this protocol supports the + given fcall. + + >>> plain.supports(100) + True + >>> plain.supports('Tversion') + True + >>> plain.supports('Rlink') + False + """ + fcall = fcall_names.get(fcall, None) + if fcall is None: + return False + cls = getattr(rrd, fcall) + return cls in self.pfods + + def get_version(self, as_bytes=True): + "get Plan 9 protocol version, as string or (default) as bytes" + ret = self.auto_vars['version'] + if as_bytes and not isinstance(ret, bytes): + ret = ret.encode('utf-8') + return ret + + @property + def version(self): + "Plan 9 protocol version" + return self.get_version(as_bytes=False) + +DEBUG = False + +# This defines a special en/decoder named "s" using a magic +# builtin. This and stat are the only variable-length +# decoders, and this is the only recursively-variable-length +# one (i.e., stat decoding is effectively fixed size once we +# handle strings). So this magic avoids the need for recursion. +# +# Note that _string_ is, in effect, size[2] orig_var[size]. +_STRING_MAGIC = '_string_' +SDesc = "typedef s: " + _STRING_MAGIC + +# This defines an en/decoder for type "qid", +# which en/decodes 1 byte called type, 4 called version, and +# 8 called path (for a total of 13 bytes). +# +# It also defines QTDIR, QTAPPEND, etc. (These are not used +# for en/decode, or at least not yet.) +QIDDesc = """\ +typedef qid: type[1] version[4] path[8] + + #define QTDIR 0x80 + #define QTAPPEND 0x40 + #define QTEXCL 0x20 + #define QTMOUNT 0x10 + #define QTAUTH 0x08 + #define QTTMP 0x04 + #define QTSYMLINK 0x02 + #define QTFILE 0x00 +""" + +# This defines a stat decoder, which has a 9p2000 standard front, +# followed by an optional additional portion. +# +# The constants are named DMDIR etc. +STATDesc = """ +typedef stat: type[2] dev[4] qid[qid] mode[4] atime[4] mtime[4] \ +length[8] name[s] uid[s] gid[s] muid[s] \ +{.u: extension[s] n_uid[4] n_gid[4] n_muid[4] } + + #define DMDIR 0x80000000 + #define DMAPPEND 0x40000000 + #define DMMOUNT 0x10000000 + #define DMAUTH 0x08000000 + #define DMTMP 0x04000000 + #define DMSYMLINK 0x02000000 + /* 9P2000.u extensions */ + #define DMDEVICE 0x00800000 + #define DMNAMEDPIPE 0x00200000 + #define DMSOCKET 0x00100000 + #define DMSETUID 0x00080000 + #define DMSETGID 0x00040000 +""" + +# This defines a wirestat decoder. A wirestat is a size and then +# a (previously encoded, or future-decoded) stat. +WirestatDesc = """ +typedef wirestat: size[2] data[size] +""" + +# This defines a dirent decoder, which has a dot-L specific format. +# +# The dirent type fields are defined as DT_* (same as BSD and Linux). +DirentDesc = """ +typedef dirent: qid[qid] offset[8] type[1] name[s] + + #define DT_UNKNOWN 0 + #define DT_FIFO 1 + #define DT_CHR 2 + #define DT_DIR 4 + #define DT_BLK 6 + #define DT_REG 8 + #define DT_LNK 10 + #define DT_SOCK 12 + #define DT_WHT 14 +""" + +# N.B.: this is largely a slightly more rigidly formatted variant of +# the contents of: +# https://github.com/chaos/diod/blob/master/protocol.md +# +# Note that <name> = <value>: ... assigns names for the fcall +# fcall (function call) table. Names without "= value" are +# assumed to be the previous value +1 (and the two names are +# also checked to make sure they are Tfoo,Rfoo). +ProtocolDesc = """\ +Rlerror.L = 7: tag[2] ecode[4] + ecode is a numerical Linux errno + +Tstatfs.L = 8: tag[2] fid[4] +Rstatfs.L: tag[2] type[4] bsize[4] blocks[8] bfree[8] bavail[8] \ + files[8] ffree[8] fsid[8] namelen[4] + Rstatfs corresponds to Linux statfs structure: + struct statfs { + long f_type; /* type of file system */ + long f_bsize; /* optimal transfer block size */ + long f_blocks; /* total data blocks in file system */ + long f_bfree; /* free blocks in fs */ + long f_bavail; /* free blocks avail to non-superuser */ + long f_files; /* total file nodes in file system */ + long f_ffree; /* free file nodes in fs */ + fsid_t f_fsid; /* file system id */ + long f_namelen; /* maximum length of filenames */ + }; + + This comes from nowhere obvious... + #define FSTYPE 0x01021997 + +Tlopen.L = 12: tag[2] fid[4] flags[4] +Rlopen.L: tag[2] qid[qid] iounit[4] + lopen prepares fid for file (or directory) I/O. + + flags contains Linux open(2) flag bits, e.g., O_RDONLY, O_RDWR, O_WRONLY. + + #define L_O_CREAT 000000100 + #define L_O_EXCL 000000200 + #define L_O_NOCTTY 000000400 + #define L_O_TRUNC 000001000 + #define L_O_APPEND 000002000 + #define L_O_NONBLOCK 000004000 + #define L_O_DSYNC 000010000 + #define L_O_FASYNC 000020000 + #define L_O_DIRECT 000040000 + #define L_O_LARGEFILE 000100000 + #define L_O_DIRECTORY 000200000 + #define L_O_NOFOLLOW 000400000 + #define L_O_NOATIME 001000000 + #define L_O_CLOEXEC 002000000 + #define L_O_SYNC 004000000 + #define L_O_PATH 010000000 + #define L_O_TMPFILE 020000000 + +Tlcreate.L = 14: tag[2] fid[4] name[s] flags[4] mode[4] gid[4] +Rlcreate.L: tag[2] qid[qid] iounit[4] + lcreate creates a regular file name in directory fid and prepares + it for I/O. + + fid initially represents the parent directory of the new file. + After the call it represents the new file. + + flags contains Linux open(2) flag bits (including O_CREAT). + + mode contains Linux creat(2) mode (permissions) bits. + + gid is the effective gid of the caller. + +Tsymlink.L = 16: tag[2] dfid[4] name[s] symtgt[s] gid[4] +Rsymlink.L: tag[2] qid[qid] + symlink creates a symbolic link name in directory dfid. The + link will point to symtgt. + + gid is the effective group id of the caller. + + The qid for the new symbolic link is returned in the reply. + +Tmknod.L = 18: tag[2] dfid[4] name[s] mode[4] major[4] minor[4] gid[4] +Rmknod.L: tag[2] qid[qid] + mknod creates a device node name in directory dfid with major + and minor numbers. + + mode contains Linux mknod(2) mode bits. (Note that these + include the S_IFMT bits which may be S_IFBLK, S_IFCHR, or + S_IFSOCK.) + + gid is the effective group id of the caller. + + The qid for the new device node is returned in the reply. + +Trename.L = 20: tag[2] fid[4] dfid[4] name[s] +Rrename.L: tag[2] + rename renames a file system object referenced by fid, to name + in the directory referenced by dfid. + + This operation will eventually be replaced by renameat. + +Treadlink.L = 22: tag[2] fid[4] +Rreadlink.L: tag[2] target[s] + readlink returns the contents of teh symbolic link referenced by fid. + +Tgetattr.L = 24: tag[2] fid[4] request_mask[8] +Rgetattr.L: tag[2] valid[8] qid[qid] mode[4] uid[4] gid[4] nlink[8] \ + rdev[8] size[8] blksize[8] blocks[8] \ + atime_sec[8] atime_nsec[8] mtime_sec[8] mtime_nsec[8] \ + ctime_sec[8] ctime_nsec[8] btime_sec[8] btime_nsec[8] \ + gen[8] data_version[8] + + getattr gets attributes of a file system object referenced by fid. + The response is intended to follow pretty closely the fields + returned by the stat(2) system call: + + struct stat { + dev_t st_dev; /* ID of device containing file */ + ino_t st_ino; /* inode number */ + mode_t st_mode; /* protection */ + nlink_t st_nlink; /* number of hard links */ + uid_t st_uid; /* user ID of owner */ + gid_t st_gid; /* group ID of owner */ + dev_t st_rdev; /* device ID (if special file) */ + off_t st_size; /* total size, in bytes */ + blksize_t st_blksize; /* blocksize for file system I/O */ + blkcnt_t st_blocks; /* number of 512B blocks allocated */ + time_t st_atime; /* time of last access */ + time_t st_mtime; /* time of last modification */ + time_t st_ctime; /* time of last status change */ + }; + + The differences are: + + * st_dev is omitted + * st_ino is contained in the path component of qid + * times are nanosecond resolution + * btime, gen and data_version fields are reserved for future use + + Not all fields are valid in every call. request_mask is a bitmask + indicating which fields are requested. valid is a bitmask + indicating which fields are valid in the response. The mask + values are as follows: + + #define GETATTR_MODE 0x00000001 + #define GETATTR_NLINK 0x00000002 + #define GETATTR_UID 0x00000004 + #define GETATTR_GID 0x00000008 + #define GETATTR_RDEV 0x00000010 + #define GETATTR_ATIME 0x00000020 + #define GETATTR_MTIME 0x00000040 + #define GETATTR_CTIME 0x00000080 + #define GETATTR_INO 0x00000100 + #define GETATTR_SIZE 0x00000200 + #define GETATTR_BLOCKS 0x00000400 + + #define GETATTR_BTIME 0x00000800 + #define GETATTR_GEN 0x00001000 + #define GETATTR_DATA_VERSION 0x00002000 + + #define GETATTR_BASIC 0x000007ff /* Mask for fields up to BLOCKS */ + #define GETATTR_ALL 0x00003fff /* Mask for All fields above */ + +Tsetattr.L = 26: tag[2] fid[4] valid[4] mode[4] uid[4] gid[4] size[8] \ + atime_sec[8] atime_nsec[8] mtime_sec[8] mtime_nsec[8] +Rsetattr.L: tag[2] + setattr sets attributes of a file system object referenced by + fid. As with getattr, valid is a bitmask selecting which + fields to set, which can be any combination of: + + mode - Linux chmod(2) mode bits. + + uid, gid - New owner, group of the file as described in Linux chown(2). + + size - New file size as handled by Linux truncate(2). + + atime_sec, atime_nsec - Time of last file access. + + mtime_sec, mtime_nsec - Time of last file modification. + + The valid bits are defined as follows: + + #define SETATTR_MODE 0x00000001 + #define SETATTR_UID 0x00000002 + #define SETATTR_GID 0x00000004 + #define SETATTR_SIZE 0x00000008 + #define SETATTR_ATIME 0x00000010 + #define SETATTR_MTIME 0x00000020 + #define SETATTR_CTIME 0x00000040 + #define SETATTR_ATIME_SET 0x00000080 + #define SETATTR_MTIME_SET 0x00000100 + + If a time bit is set without the corresponding SET bit, the + current system time on the server is used instead of the value + sent in the request. + +Txattrwalk.L = 30: tag[2] fid[4] newfid[4] name[s] +Rxattrwalk.L: tag[2] size[8] + xattrwalk gets a newfid pointing to xattr name. This fid can + later be used to read the xattr value. If name is NULL newfid + can be used to get the list of extended attributes associated + with the file system object. + +Txattrcreate.L = 32: tag[2] fid[4] name[s] attr_size[8] flags[4] +Rxattrcreate.L: tag[2] + xattrcreate gets a fid pointing to the xattr name. This fid + can later be used to set the xattr value. + + flag is derived from set Linux setxattr. The manpage says + + The flags parameter can be used to refine the semantics of + the operation. XATTR_CREATE specifies a pure create, + which fails if the named attribute exists already. + XATTR_REPLACE specifies a pure replace operation, which + fails if the named attribute does not already exist. By + default (no flags), the extended attribute will be created + if need be, or will simply replace the value if the + attribute exists. + + The actual setxattr operation happens when the fid is clunked. + At that point the written byte count and the attr_size + specified in TXATTRCREATE should be same otherwise an error + will be returned. + +Treaddir.L = 40: tag[2] fid[4] offset[8] count[4] +Rreaddir.L: tag[2] count[4] data[count] + readdir requests that the server return directory entries from + the directory represented by fid, previously opened with + lopen. offset is zero on the first call. + + Directory entries are represented as variable-length records: + qid[qid] offset[8] type[1] name[s] + At most count bytes will be returned in data. If count is not + zero in the response, more data is available. On subsequent + calls, offset is the offset returned in the last directory + entry of the previous call. + +Tfsync.L = 50: tag[2] fid[4] +Rfsync.L: tag[2] + fsync tells the server to flush any cached data associated + with fid, previously opened with lopen. + +Tlock.L = 52: tag[2] fid[4] type[1] flags[4] start[8] length[8] \ + proc_id[4] client_id[s] +Rlock.L: tag[2] status[1] + lock is used to acquire or release a POSIX record lock on fid + and has semantics similar to Linux fcntl(F_SETLK). + + type has one of the values: + + #define LOCK_TYPE_RDLCK 0 + #define LOCK_TYPE_WRLCK 1 + #define LOCK_TYPE_UNLCK 2 + + start, length, and proc_id correspond to the analagous fields + passed to Linux fcntl(F_SETLK): + + struct flock { + short l_type; /* Type of lock: F_RDLCK, F_WRLCK, F_UNLCK */ + short l_whence;/* How to intrprt l_start: SEEK_SET,SEEK_CUR,SEEK_END */ + off_t l_start; /* Starting offset for lock */ + off_t l_len; /* Number of bytes to lock */ + pid_t l_pid; /* PID of process blocking our lock (F_GETLK only) */ + }; + + flags bits are: + + #define LOCK_SUCCESS 0 + #define LOCK_BLOCKED 1 + #define LOCK_ERROR 2 + #define LOCK_GRACE 3 + + The Linux v9fs client implements the fcntl(F_SETLKW) + (blocking) lock request by calling lock with + LOCK_FLAGS_BLOCK set. If the response is LOCK_BLOCKED, + it retries the lock request in an interruptible loop until + status is no longer LOCK_BLOCKED. + + The Linux v9fs client translates BSD advisory locks (flock) to + whole-file POSIX record locks. v9fs does not implement + mandatory locks and will return ENOLCK if use is attempted. + + Because of POSIX record lock inheritance and upgrade + properties, pass-through servers must be implemented + carefully. + +Tgetlock.L = 54: tag[2] fid[4] type[1] start[8] length[8] proc_id[4] \ + client_id[s] +Rgetlock.L: tag[2] type[1] start[8] length[8] proc_id[4] client_id[s] + getlock tests for the existence of a POSIX record lock and has + semantics similar to Linux fcntl(F_GETLK). + + As with lock, type has one of the values defined above, and + start, length, and proc_id correspond to the analagous fields + in struct flock passed to Linux fcntl(F_GETLK), and client_Id + is an additional mechanism for uniquely identifying the lock + requester and is set to the nodename by the Linux v9fs client. + +Tlink.L = 70: tag[2] dfid[4] fid[4] name[s] +Rlink.L: tag[2] + link creates a hard link name in directory dfid. The link + target is referenced by fid. + +Tmkdir.L = 72: tag[2] dfid[4] name[s] mode[4] gid[4] +Rmkdir.L: tag[2] qid[qid] + mkdir creates a new directory name in parent directory dfid. + + mode contains Linux mkdir(2) mode bits. + + gid is the effective group ID of the caller. + + The qid of the new directory is returned in the response. + +Trenameat.L = 74: tag[2] olddirfid[4] oldname[s] newdirfid[4] newname[s] +Rrenameat.L: tag[2] + Change the name of a file from oldname to newname, possible + moving it from old directory represented by olddirfid to new + directory represented by newdirfid. + + If the server returns ENOTSUPP, the client should fall back to + the rename operation. + +Tunlinkat.L = 76: tag[2] dirfd[4] name[s] flags[4] +Runlinkat.L: tag[2] + Unlink name from directory represented by dirfd. If the file + is represented by a fid, that fid is not clunked. If the + server returns ENOTSUPP, the client should fall back to the + remove operation. + + There seems to be only one defined flag: + + #define AT_REMOVEDIR 0x200 + +Tversion = 100: tag[2] msize[4] version[s]:auto +Rversion: tag[2] msize[4] version[s] + + negotiate protocol version + + version establishes the msize, which is the maximum message + size inclusive of the size value that can be handled by both + client and server. + + It also establishes the protocol version. For 9P2000.L + version must be the string 9P2000.L. + +Tauth = 102: tag[2] afid[4] uname[s] aname[s] n_uname[4] +Rauth: tag[2] aqid[qid] + auth initiates an authentication handshake for n_uname. + Rlerror is returned if authentication is not required. If + successful, afid is used to read/write the authentication + handshake (protocol does not specify what is read/written), + and afid is presented in the attach. + +Tattach = 104: tag[2] fid[4] afid[4] uname[s] aname[s] {.u: n_uname[4] } +Rattach: tag[2] qid[qid] + attach introduces a new user to the server, and establishes + fid as the root for that user on the file tree selected by + aname. + + afid can be NOFID (~0) or the fid from a previous auth + handshake. The afid can be clunked immediately after the + attach. + + #define NOFID 0xffffffff + + n_uname, if not set to NONUNAME (~0), is the uid of the + user and is used in preference to uname. Note that it appears + in both .u and .L (unlike most .u-specific features). + + #define NONUNAME 0xffffffff + + v9fs has several modes of access which determine how it uses + attach. In the default access=user, an initial attach is sent + for the user provided in the uname=name mount option, and for + each user that accesses the file system thereafter. For + access=, only the initial attach is sent for and all other + users are denied access by the client. + +Rerror = 107: tag[2] errstr[s] {.u: errnum[4] } + +Tflush = 108: tag[2] oldtag[2] +Rflush: tag[2] + flush aborts an in-flight request referenced by oldtag, if any. + +Twalk = 110: tag[2] fid[4] newfid[4] nwname[2] nwname*(wname[s]) +Rwalk: tag[2] nwqid[2] nwqid*(wqid[qid]) + walk is used to descend a directory represented by fid using + successive path elements provided in the wname array. If + succesful, newfid represents the new path. + + fid can be cloned to newfid by calling walk with nwname set to + zero. + + if nwname==0, fid need not represent a directory. + +Topen = 112: tag[2] fid[4] mode[1] +Ropen: tag[2] qid[qid] iounit[4] + open prepares fid for file (or directory) I/O. + + mode is: + #define OREAD 0 /* open for read */ + #define OWRITE 1 /* open for write */ + #define ORDWR 2 /* open for read and write */ + #define OEXEC 3 /* open for execute */ + + #define OTRUNC 16 /* truncate (illegal if OEXEC) */ + #define OCEXEC 32 /* close on exec (nonsensical) */ + #define ORCLOSE 64 /* remove on close */ + #define ODIRECT 128 /* direct access (.u extension?) */ + +Tcreate = 114: tag[2] fid[4] name[s] perm[4] mode[1] {.u: extension[s] } +Rcreate: tag[2] qid[qid] iounit[4] + create is similar to open; however, the incoming fid is the + diretory in which the file is to be created, and on success, + return, the fid refers to the then-created file. + +Tread = 116: tag[2] fid[4] offset[8] count[4] +Rread: tag[2] count[4] data[count] + perform a read on the file represented by fid. Note that in + v9fs, a read(2) or write(2) system call for a chunk of the + file that won't fit in a single request is broken up into + multiple requests. + + Under 9P2000.L, read cannot be used on directories. See readdir. + +Twrite = 118: tag[2] fid[4] offset[8] count[4] data[count] +Rwrite: tag[2] count[4] + perform a write on the file represented by fid. Note that in + v9fs, a read(2) or write(2) system call for a chunk of the + file that won't fit in a single request is broken up into + multiple requests. + + write cannot be used on directories. + +Tclunk = 120: tag[2] fid[4] +Rclunk: tag[2] + clunk signifies that fid is no longer needed by the client. + +Tremove = 122: tag[2] fid[4] +Rremove: tag[2] + remove removes the file system object represented by fid. + + The fid is always clunked (even on error). + +Tstat = 124: tag[2] fid[4] +Rstat: tag[2] size[2] data[size] + +Twstat = 126: tag[2] fid[4] size[2] data[size] +Rwstat: tag[2] +""" + +class _Token(object): + r""" + A scanned token. + + Tokens have a type (tok.ttype) and value (tok.value). The value + is generally the token itself, although sometimes a prefix and/or + suffix has been removed (for 'label', 'word*', ':aux', and + '[type]' tokens). If prefix and/or suffix are removed, the full + original token is + in its .orig. + + Tokens are: + - 'word', 'word*', or 'label': + '[.\w]+' followed by optional '*' or ':': + + - 'aux': ':' followed by '\w+' (used for :auto annotation) + + - 'type': + open bracket '[', followed by '\w+' or '\d+' (only one of these), + followed by close bracket ']' + + - '(', ')', '{', '}': themeselves + + Each token can have arbitrary leading white space (which is + discarded). + + (Probably should return ':' as a char and handle it in parser, + but oh well.) + """ + def __init__(self, ttype, value, orig=None): + self.ttype = ttype + self.value = value + self.orig = value if orig is None else orig + if self.ttype == 'type' and self.value.isdigit(): + self.ival = int(self.value) + else: + self.ival = None + def __str__(self): + return self.orig + +_Token.tok_expr = re.compile(r'\s*([.\w]+(?:\*|:)?' + r'|:\w+' + r'|\[(?:\w+|\d+)\]' + r'|[(){}])') + +def _scan(string): + """ + Tokenize a string. + + Note: This raises a ValueError with the position of any unmatched + character in the string. + """ + tlist = [] + + # make sure entire string is tokenized properly + pos = 0 + for item in _Token.tok_expr.finditer(string): + span = item.span() + if span[0] != pos: + print('error: unmatched character(s) in input\n{0}\n{1}^'.format( + string, ' ' * pos)) + raise ValueError('unmatched lexeme', pos) + pos = span[1] + tlist.append(item.group(1)) + if pos != len(string): + print('error: unmatched character(s) in input\n{0}\n{1}^'.format( + string, ' ' * pos)) + raise ValueError('unmatched lexeme', pos) + + # classify each token, stripping decorations + result = [] + for item in tlist: + if item in ('(', ')', '{', '}'): + tok = _Token(item, item) + elif item[0] == ':': + tok = _Token('aux', item[1:], item) + elif item.endswith(':'): + tok = _Token('label', item[0:-1], item) + elif item.endswith('*'): + tok = _Token('word*', item[0:-1], item) + elif item[0] == '[': + # integer or named type + if item[-1] != ']': + raise ValueError('internal error: "{0}" is not [...]'.format( + item)) + tok = _Token('type', item[1:-1], item) + else: + tok = _Token('word', item) + result.append(tok) + return result + +def _debug_print_sequencer(seq): + """for debugging""" + print('sequencer is {0!r}'.format(seq), file=sys.stderr) + for i, enc in enumerate(seq): + print(' [{0:d}] = {1}'.format(i, enc), file=sys.stderr) + +def _parse_expr(seq, string, typedefs): + """ + Parse "expression-ish" items, which is a list of: + name[type] + name*(subexpr) (a literal asterisk) + { label ... } + + The "type" may be an integer or a second name. In the case + of a second name it must be something from <typedefs>. + + The meaning of name[integer] is that we are going to encode + or decode a fixed-size field of <integer> bytes, using the + given name. + + For name[name2], we can look up name2 in our typedefs table. + The only real typedefs's used here are "stat" and "s"; each + of these expands to a variable-size encode/decode. See the + special case below, though. + + The meaning of name*(...) is: the earlier name will have been + defined by an earlier _parse_expr for this same line. That + earlier name provides a repeat-count. + + Inside the parens we get a name[type] sub-expressino. This may + not recurse further, so we can use a pretty cheesy parser. + + As a special case, given name[name2], we first check whether + name2 is an earlier name a la name*(...). Here the meaning + is much like name2*(name[1]), except that the result is a + simple byte string, rather than an array. + + The meaning of "{ label ... " is that everything following up + to "}" is optional and used only with 9P2000.u and/or 9P2000.L. + Inside the {...} pair is the usual set of tokens, but again + {...} cannot recurse. + + The parse fills in a Sequencer instance, and returns a list + of the parsed names. + """ + names = [] + cond = None + + tokens = collections.deque(_scan(string)) + + def get_subscripted(tokens): + """ + Allows name[integer] and name1[name2] only; returns + tuple after stripping off both tokens, or returns None + and does not strip tokens. + """ + if len(tokens) == 0 or tokens[0].ttype != 'word': + return None + if len(tokens) > 1 and tokens[1].ttype == 'type': + word = tokens.popleft() + return word, tokens.popleft() + return None + + def lookup(name, typeinfo, aux=None): + """ + Convert cond (if not None) to its .value, so that instead + of (x, '.u') we get '.u'. + + Convert typeinfo to an encdec. Typeinfo may be 1/2/4/8, or + one of our typedef names. If it's a typedef name it will + normally correspond to an EncDecTyped, but we have one special + case for string types, and another for using an earlier-defined + variable. + """ + condval = None if cond is None else cond.value + if typeinfo.ival is None: + try: + cls, sub = typedefs[typeinfo.value] + except KeyError: + raise ValueError('unknown type name {0}'.format(typeinfo)) + # the type name is typeinfo.value; the corresponding + # pfod class is cls; the *variable* name is name; + # and the sub-sequence is sub. But if cls is None + # then it's our string type. + if cls is None: + encdec = sequencer.EncDecSimple(name, _STRING_MAGIC, aux) + else: + encdec = sequencer.EncDecTyped(cls, name, sub, aux) + else: + if typeinfo.ival not in (1, 2, 4, 8): + raise ValueError('bad integer code in {0}'.format(typeinfo)) + encdec = sequencer.EncDecSimple(name, typeinfo.ival, aux) + return condval, encdec + + def emit_simple(name, typeinfo, aux=None): + """ + Emit name[type]. We may be inside a conditional; if so + cond is not None. + """ + condval, encdec = lookup(name, typeinfo, aux) + seq.append_encdec(condval, encdec) + names.append(name) + + def emit_repeat(name1, name2, typeinfo): + """ + Emit name1*(name2[type]). + + Note that the conditional is buried in the sub-coder for + name2. It must be passed through anyway in case the sub- + coder is only partly conditional. If the sub-coder is + fully conditional, each sub-coding uses or produces no + bytes and hence the array itself is effectively conditional + as well (it becomes name1 * [None]). + + We don't (currently) have any auxiliary data for arrays. + """ + if name1 not in names: + raise ValueError('{0}*({1}[{2}]): ' + '{0} undefined'.format(name1, name2, + typeinfo.value)) + condval, encdec = lookup(name2, typeinfo) + encdec = sequencer.EncDecA(name1, name2, encdec) + seq.append_encdec(condval, encdec) + names.append(name2) + + def emit_bytes_repeat(name1, name2): + """ + Emit name1[name2], e.g., data[count]. + """ + condval = None if cond is None else cond.value + # Note that the two names are reversed when compared to + # count*(data[type]). The "sub-coder" is handled directly + # by EncDecA, hence is None. + # + # As a peculiar side effect, all bytes-repeats cause the + # count itself to become automatic (to have an aux of 'len'). + encdec = sequencer.EncDecA(name2, name1, None, 'len') + seq.append_encdec(condval, encdec) + names.append(name1) + + supported_conditions = ('.u') + while tokens: + token = tokens.popleft() + if token.ttype == 'label': + raise ValueError('misplaced label') + if token.ttype == 'aux': + raise ValueError('misplaced auxiliary') + if token.ttype == '{': + if cond is not None: + raise ValueError('nested "{"') + if len(tokens) == 0: + raise ValueError('unclosed "{"') + cond = tokens.popleft() + if cond.ttype != 'label': + raise ValueError('"{" not followed by cond label') + if cond.value not in supported_conditions: + raise ValueError('unsupported condition "{0}"'.format( + cond.value)) + continue + if token.ttype == '}': + if cond is None: + raise ValueError('closing "}" w/o opening "{"') + cond = None + continue + if token.ttype == 'word*': + if len(tokens) == 0 or tokens[0].ttype != '(': + raise ValueError('{0} not followed by (...)'.format(token)) + tokens.popleft() + repeat = get_subscripted(tokens) + if repeat is None: + raise ValueError('parse error after {0}('.format(token)) + if len(tokens) == 0 or tokens[0].ttype != ')': + raise ValueError('missing ")" after {0}({1}{2}'.format( + token, repeat[0], repeat[1])) + tokens.popleft() + # N.B.: a repeat cannot have an auxiliary info (yet?). + emit_repeat(token.value, repeat[0].value, repeat[1]) + continue + if token.ttype == 'word': + # Special case: _STRING_MAGIC turns into a string + # sequencer. This should be used with just one + # typedef (typedef s: _string_). + if token.value == _STRING_MAGIC: + names.append(_STRING_MAGIC) # XXX temporary + continue + if len(tokens) == 0 or tokens[0].ttype != 'type': + raise ValueError('parse error after {0}'.format(token)) + type_or_size = tokens.popleft() + # Check for name[name2] where name2 is a word (not a + # number) that is in the names[] array. + if type_or_size.value in names: + # NB: this cannot have auxiliary info. + emit_bytes_repeat(token.value, type_or_size.value) + continue + if len(tokens) > 0 and tokens[0].ttype == 'aux': + aux = tokens.popleft() + if aux.value != 'auto': + raise ValueError('{0}{1}: only know "auto", not ' + '{2}'.format(token, type_or_size, + aux.value)) + emit_simple(token.value, type_or_size, aux.value) + else: + emit_simple(token.value, type_or_size) + continue + raise ValueError('"{0}" not valid here"'.format(token)) + + if cond is not None: + raise ValueError('unclosed "}"') + + return names + +class _ProtoDefs(object): + def __init__(self): + # Scan our typedefs. This may execute '#define's as well. + self.typedefs = {} + self.defines = {} + typedef_re = re.compile(r'\s*typedef\s+(\w+)\s*:\s*(.*)') + self.parse_lines('SDesc', SDesc, typedef_re, self.handle_typedef) + self.parse_lines('QIDDesc', QIDDesc, typedef_re, self.handle_typedef) + self.parse_lines('STATDesc', STATDesc, typedef_re, self.handle_typedef) + self.parse_lines('WirestatDesc', WirestatDesc, typedef_re, + self.handle_typedef) + self.parse_lines('DirentDesc', DirentDesc, typedef_re, + self.handle_typedef) + + # Scan protocol (the bulk of the work). This, too, may + # execute '#define's. + self.protocol = {} + proto_re = re.compile(r'(\*?\w+)(\.\w+)?\s*(?:=\s*(\d+))?\s*:\s*(.*)') + self.prev_proto_value = None + self.parse_lines('ProtocolDesc', ProtocolDesc, + proto_re, self.handle_proto_def) + + self.setup_header() + + # set these up for export() + self.plain = {} + self.dotu = {} + self.dotl = {} + + def parse_lines(self, name, text, regexp, match_handler): + """ + Parse a sequence of lines. Match each line using the + given regexp, or (first) as a #define line. Note that + indented lines are either #defines or are commentary! + + If hnadling raises a ValueError, we complain and include + the appropriate line offset. Then we sys.exit(1) (!). + """ + define = re.compile(r'\s*#define\s+(\w+)\s+([^/]*)' + r'(\s*/\*.*\*/)?\s*$') + for lineoff, line in enumerate(text.splitlines()): + try: + match = define.match(line) + if match: + self.handle_define(*match.groups()) + continue + match = regexp.match(line) + if match: + match_handler(*match.groups()) + continue + if len(line) and not line[0].isspace(): + raise ValueError('unhandled line: {0}'.format(line)) + except ValueError as err: + print('Internal error while parsing {0}:\n' + ' {1}\n' + '(at line offset +{2}, discounting \\-newline)\n' + 'The original line in question reads:\n' + '{3}'.format(name, err.args[0], lineoff, line), + file=sys.stderr) + sys.exit(1) + + def handle_define(self, name, value, comment): + """ + Handle #define match. + + The regexp has three fields, matching the name, value, + and possibly-empty comment; these are our arguments. + """ + # Obnoxious: int(,0) requires new 0o syntax in py3k; + # work around by trying twice, once with base 0, then again + # with explicit base 8 if the first attempt fails. + try: + value = int(value, 0) + except ValueError: + value = int(value, 8) + if DEBUG: + print('define: defining {0} as {1:x}'.format(name, value), + file=sys.stderr) + if name in self.defines: + raise ValueError('redefining {0}'.format(name)) + self.defines[name] = (value, comment) + + def handle_typedef(self, name, expr): + """ + Handle typedef match. + + The regexp has just two fields, the name and the expression + to parse (note that the expression must fit all on one line, + using backslach-newline if needed). + + Typedefs may refer back to existing typedefs, so we pass + self.typedefs to _parse_expr(). + """ + seq = sequencer.Sequencer(name) + fields = _parse_expr(seq, expr, self.typedefs) + # Check for special string magic typedef. (The name + # probably should be just 's' but we won't check that + # here.) + if len(fields) == 1 and fields[0] == _STRING_MAGIC: + cls = None + else: + cls = pfod.pfod(name, fields) + if DEBUG: + print('typedef: {0} = {1!r}; '.format(name, fields), + end='', file=sys.stderr) + _debug_print_sequencer(seq) + if name in self.typedefs: + raise ValueError('redefining {0}'.format(name)) + self.typedefs[name] = cls, seq + + def handle_proto_def(self, name, proto_version, value, expr): + """ + Handle protocol definition. + + The regexp matched: + - The name of the protocol option such as Tversion, + Rversion, Rlerror, etc. + - The protocol version, if any (.u or .L). + - The value, if specified. If no value is specified + we use "the next value". + - The expression to parse. + + As with typedefs, the expression must fit all on one + line. + """ + if value: + value = int(value) + elif self.prev_proto_value is not None: + value = self.prev_proto_value + 1 + else: + raise ValueError('{0}: missing protocol value'.format(name)) + if value < 0 or value > 255: + raise ValueError('{0}: protocol value {1} out of ' + 'range'.format(name, value)) + self.prev_proto_value = value + + seq = sequencer.Sequencer(name) + fields = _parse_expr(seq, expr, self.typedefs) + cls = pfod.pfod(name, fields) + if DEBUG: + print('proto: {0} = {1}; '.format(name, value), + end='', file=sys.stderr) + _debug_print_sequencer(seq) + if name in self.protocol: + raise ValueError('redefining {0}'.format(name)) + self.protocol[name] = cls, value, proto_version, seq + + def setup_header(self): + """ + Handle header definition. + + This is a bit gimmicky and uses some special cases, + because data is sized to dsize which is effectively + just size - 5. We can't express this in our mini language, + so we just hard-code the sequencer and pfod. + + In addition, the unpacker never gets the original packet's + size field, only the fcall and the data. + """ + self.header_pfod = pfod.pfod('Header', 'size dsize fcall data') + + seq = sequencer.Sequencer('Header-pack') + # size: 4 bytes + seq.append_encdec(None, sequencer.EncDecSimple('size', 4, None)) + # fcall: 1 byte + seq.append_encdec(None, sequencer.EncDecSimple('fcall', 1, None)) + # data: string of length dsize + seq.append_encdec(None, sequencer.EncDecA('dsize', 'data', None)) + if DEBUG: + print('Header-pack:', file=sys.stderr) + _debug_print_sequencer(seq) + self.header_pack_seq = seq + + seq = sequencer.Sequencer('Header-unpack') + seq.append_encdec(None, sequencer.EncDecSimple('fcall', 1, None)) + seq.append_encdec(None, sequencer.EncDecA('dsize', 'data', None)) + if DEBUG: + print('Header-unpack:', file=sys.stderr) + _debug_print_sequencer(seq) + self.header_unpack_seq = seq + + def export(self, mod): + """ + Dump results of internal parsing process + into our module namespace. + + Note that we do not export the 's' typedef, which + did not define a data structure. + + Check for name collisions while we're at it. + """ + namespace = type('td', (object,), {}) + + # Export the typedefs (qid, stat). + setattr(mod, 'td', namespace) + for key in self.typedefs: + cls = self.typedefs[key][0] + if cls is None: + continue + setattr(namespace, key, cls) + + # Export two sequencers for en/decoding stat fields + # (needed for reading directories and doing Twstat). + setattr(namespace, 'stat_seq', self.typedefs['stat'][1]) + setattr(namespace, 'wirestat_seq', self.typedefs['wirestat'][1]) + + # Export the similar dirent decoder. + setattr(namespace, 'dirent_seq', self.typedefs['dirent'][1]) + + # Export the #define values + for key, val in self.defines.items(): + if hasattr(namespace, key): + print('{0!r} is both a #define and a typedef'.format(key)) + raise AssertionError('bad internal names') + setattr(namespace, key, val[0]) + + # Export Tattach, Rattach, Twrite, Rversion, etc values. + # Set up fcall_names[] table to map from value back to name. + # We also map fcall names to themselves, so given either a + # name or a byte code we can find out whether it's a valid + # fcall. + for key, val in self.protocol.items(): + if hasattr(namespace, key): + prev_def = '#define' if key in self.defines else 'typedef' + print('{0!r} is both a {1} and a protocol ' + 'value'.format(key, prev_def)) + raise AssertionError('bad internal names') + setattr(namespace, key, val[1]) + fcall_names[key] = key + fcall_names[val[1]] = key + + # Hook up PFOD's for each protocol object -- for + # Tversion/Rversion, Twrite/Rwrite, Tlopen/Rlopen, etc. + # They go in the rrd name-space, and also in dictionaries + # per-protocol here, with the lookup pointing to a _PackInfo + # for the corresponding sequencer. + # + # Note that each protocol PFOD is optionally annotated with + # its specific version. We know that .L > .u > plain; but + # all the "lesser" PFODs are available to all "greater" + # protocols at all times. + # + # (This is sort-of-wrong for Rerror vs Rlerror, but we + # don't bother to exclude Rerror from .L.) + # + # The PFODs themselves were already created, at parse time. + namespace = type('rrd', (object,), {}) + setattr(mod, 'rrd', namespace) + for key, val in self.protocol.items(): + cls = val[0] + proto_version = val[2] + seq = val[3] + packinfo = _PackInfo(seq) + if proto_version is None: + # all three protocols have it + self.plain[cls] = packinfo + self.dotu[cls] = packinfo + self.dotl[cls] = packinfo + elif proto_version == '.u': + # only .u and .L have it + self.dotu[cls] = packinfo + self.dotl[cls] = packinfo + elif proto_version == '.L': + # only .L has it + self.dotl[cls] = packinfo + else: + raise AssertionError('unknown protocol {1} for ' + '{0}'.format(key, proto_version)) + setattr(namespace, key, cls) + +_9p_data = _ProtoDefs() +_9p_data.export(sys.modules[__name__]) + +# Currently we look up by text-string, in lowercase. +_9p_versions = { + '9p2000': _P9Proto({'version': '9P2000'}, + {'.u': False}, + _9p_data, + _9p_data.plain, + 0), + '9p2000.u': _P9Proto({'version': '9P2000.u'}, + {'.u': True}, + _9p_data, + _9p_data.dotu, + 1), + '9p2000.l': _P9Proto({'version': '9P2000.L'}, + {'.u': True}, + _9p_data, + _9p_data.dotl, + 2), +} +def p9_version(vers_string): + """ + Return protocol implementation of given version. Raises + KeyError if the version is invalid. Note that the KeyError + will be on a string-ified, lower-cased version of the vers_string + argument, even if it comes in as a bytes instance in py3k. + """ + if not isinstance(vers_string, str) and isinstance(vers_string, bytes): + vers_string = vers_string.decode('utf-8', 'surrogateescape') + return _9p_versions[vers_string.lower()] + +plain = p9_version('9p2000') +dotu = p9_version('9p2000.u') +dotl = p9_version('9p2000.L') + +def qid_type2name(qidtype): + """ + Convert qid type field to printable string. + + >>> qid_type2name(td.QTDIR) + 'dir' + >>> qid_type2name(td.QTAPPEND) + 'append-only' + >>> qid_type2name(0xff) + 'invalid(0xff)' + """ + try: + # Is it ever OK to have multiple bits set, + # e.g., both QTAPPEND and QTEXCL? + return { + td.QTDIR: 'dir', + td.QTAPPEND: 'append-only', + td.QTEXCL: 'exclusive', + td.QTMOUNT: 'mount', + td.QTAUTH: 'auth', + td.QTTMP: 'tmp', + td.QTSYMLINK: 'symlink', + td.QTFILE: 'file', + }[qidtype] + except KeyError: + pass + return 'invalid({0:#x})'.format(qidtype) + +if __name__ == '__main__': + import doctest + doctest.testmod() |