Tuesday, July 24, 2012

Introducing auto_argument

One of the subtle bugs that often bites new Python programmers is default argument values. People expect those default values to be (re-)created and supplied to the function at each invocation. And so, when it does not happen that way, developers scratch their heads.

It is easy, though, to understand why Python acts that way. As explained in the Python Tutorial:

The default values are evaluated at the point of function definition in the defining scope.

And that:

The default value is evaluated only once. This makes a difference when the default is a mutable object such as a list, dictionary, or instances of most classes.

To prevent the default from being shared between subsequent calls, one usually writes code like this:

def f(a, L=None):
    if L is None:
        L = []
    # do something with L

The if block creates a new actual default value to replace the None marker value.

To alleviate repeated typings and clutter if blocks, I wrote a simple module auto_argument that automatically inspects the function arguments on each invocation and replaces them with new values. The code above would look cleaner with callable_auto_argument like this:

@callable_auto_argument([('L', None, dict)])
def f(a, L=None):
    # do something with L

Of course, developers are very welcome to make their own decorator by subclassing one of these auto_argument classes to save redundant code. See auto_dict_argument and test_subclass in the source code for example.

And here is auto_argument in all its glory :-). The code is released under MIT license.

# Copyright 2012, Nam T. Nguyen
# Released under MIT license

'''Automatically replace a default argument with some other (potentially
dynamic) value.

The default argument is usually guarded like this::

    def func(arg=None):
        if arg is None:
            arg = dict()
        // use arg

With decorators provided in this module, one can write::

    __marker = object()

    @callable_auto_argument([('arg', __marker, dict)])
    def func(arg=__marker):
        // use arg

See class:`callable_auto_argument`.

Also, the standard Python behavior could be thought as::

    __marker = object()

    @passthru_auto_argument([('arg', __marker, {})])
    def func(arg=__marker):
        // ...

See class:`passthru_auto_argument`.

These classes can be used by themselves or serve as base classes for more
customizations. For example, to eliminate repeated typings, one can subclass
``callable_auto_argument`` like this::

    class auto_dict_argument(callable_auto_argument):

        def __init__(self, *names):
            names_markers_values = []
            for name in names:
                names_markers_values.append((name, None, dict))
            super(auto_dict_argument, self).__init__(names_markers_values)

And then apply this on methods like this::

    @auto_dict_argument('arg_1', 'arg_2', 'arg_3')
    def method(arg_1=None, arg_2=None, arg_3=None):
        # arg_1, arg_2, arg_3 are new dicts unless specified otherwise
        # and these lines are no longer required
        # if arg_1 is None: arg_1 = {}
        # if arg_2 is None: arg_2 = {}
        # if arg_3 is None: arg_3 = {}


import unittest

class auto_argument(object):
    '''The base class for automatic argument.

    Subclasses must implement method:`create` to create appropriate value for
    the argument.


    def __init__(self, names_markers_values):
        '''Construct an auto argument decorator with a collection of variable
        names, their markers, and their supposed values.

        The __supposed__ value objects are used in method:`create` to produce
        final value.


            names_markers_values (collection of 3-tuples): A collection of
                (string, object, object) tuples specifying (in that order) the
                names, the marker objects, and the supposed value objects


        self.names_markers_values = names_markers_values

    def create(self, name, current_value, value):
        '''Return a value based for the named argument and its current value.

        This final value will be used to replace what is currently passed in
        the invocation.

        Subclasses MUST override this method to provide more meaningful


            name (string): The argument's name
            current_value: Its current value in this invocation
            value: The supposed value passed in during construction time


            Final value for this argument


        raise NotImplementedError()

    def __call__(self, orig_func):
        def new_func(*args, **kw_args):
            for name, marker, value in self.names_markers_values:
                # check kw_args first
                    if kw_args[name] is marker:
                        kw_args[name] = self.create(name, kw_args[name], value)
                except KeyError:
                    # ignored

                # then check args
                # we need to instropect the arg names from orig_func
                co_obj = orig_func.func_code
                # number of required arguments
                nr_required = (co_obj.co_argcount -
                for i in range(nr_required, co_obj.co_argcount):
                    if co_obj.co_varnames[i] != name:
                    # is it supplied in args?
                    if i < len(args):
                        if args[i] is marker:
                            if isinstance(args, tuple):
                                args = list(args)
                            args[i] = self.create(name, args[i], value)
                    # it is not, so, check defaults
                        default = orig_func.func_defaults[i - nr_required]
                        if default is marker:
                            kw_args[name] = self.create(name, default, value)

            # invoke orig_func with new args
            return orig_func(*args, **kw_args)

        return new_func

class callable_auto_argument(auto_argument):

    def create(self, name, current_value, value):
        # call on value
        return value()

class passthru_auto_argument(auto_argument):

    def create(self, name, current_value, value):
        # just return it directly
        return value

class AutoArgumentTest(unittest.TestCase):

    def test_keyword_1(self):

        marker = 'replace_me'

        @passthru_auto_argument([('arg_name', marker, 'done')])
        def orig_func_0(arg_name=marker):
            return arg_name

        self.assertEqual('done', orig_func_0())
        self.assertEqual('done', orig_func_0(marker))
        self.assertEqual('not_replace', orig_func_0('not_replace'))
        self.assertEqual('done', orig_func_0(arg_name=marker))
        self.assertEqual('not_replace', orig_func_0(arg_name='not_replace'))

        @passthru_auto_argument([('arg_name', 'replace_me', 'done')])
        def orig_func_1(junk, arg_name='replace_me'):
            return junk, arg_name

        self.assertEqual(('ignore', 'done'), orig_func_1('ignore'))
        self.assertEqual(('ignore', 'done'),
                orig_func_1('ignore', marker))
        self.assertEqual(('ignore', 'not_replace'),
                orig_func_1('ignore', 'not_replace'))
        self.assertEqual(('ignore', 'done'),
                orig_func_1('ignore', arg_name=marker))
        self.assertEqual(('ignore', 'not_replace'),
                orig_func_1('ignore', arg_name='not_replace'))

    def test_keyword_2(self):

        marker_1 = 'replace_me'
        marker_2 = 'replace_too'

        @passthru_auto_argument([('arg_1', marker_1, 'done'),
                     ('arg_2', marker_2, 'enod')])
        def orig_func_0(arg_1=marker_1, arg_2=marker_2):
            return arg_1, arg_2

        self.assertEqual(('done', 'enod'), orig_func_0())
        self.assertEqual(('not_replace', 'enod'), orig_func_0('not_replace'))
        self.assertEqual(('done', 'not'), orig_func_0(marker_1, 'not'))
        self.assertEqual(('done', 'enod'), orig_func_0(marker_1, marker_2))
        self.assertEqual(('not_1', 'not_2'), orig_func_0('not_1', 'not_2'))

        @passthru_auto_argument([('arg_1', marker_1, 'done'),
                     ('arg_2', marker_2, 'enod')])
        def orig_func_1(junk, arg_1=marker_1, arg_2=marker_2):
            return junk, arg_1, arg_2

        self.assertEqual(('.', 'done', 'enod'), orig_func_1('.'))
        self.assertEqual(('.', 'not_replace', 'enod'),
                orig_func_1('.', 'not_replace'))
        self.assertEqual(('.', 'done', 'not'),
                orig_func_1('.', marker_1, 'not'))
        self.assertEqual(('.', 'done', 'enod'),
                orig_func_1('.', marker_1, marker_2))
        self.assertEqual(('.', 'not_1', 'not_2'),
                orig_func_1('.', 'not_1', 'not_2'))

    def test_subclass(self):

        class auto_dict_argument(callable_auto_argument):

            def __init__(self, *names):
                names_markers_values = []
                for name in names:
                    names_markers_values.append((name, None, dict))
                super(auto_dict_argument, self).__init__(names_markers_values)

        @auto_dict_argument('arg_1', 'arg_2')
        def test_func(arg_1=None, arg_2=None):
            arg_1['1'] = 'arg_1'
            arg_2['2'] = 'arg_2'
            return (arg_1, arg_2)

        self.assertEqual(({'1': 'arg_1'}, {'2': 'arg_2'}), test_func())
        arg_1, arg_2 = test_func()
        self.assertNotEqual(id(arg_1), id(arg_2))

if __name__ == '__main__':

Friday, July 20, 2012

Interface programming with intfprgm

Based on the blog post from two days ago, I continued to make two more additions to the set of decorators. The resulting module is called intfprgm, for interface programming. The repository is hosted at https://bitbucket.org/namn/intfprgm.

A careful reader will notice that the LICENSE file is actually MIT license when the source is released to the public domain. I'd like to confirm that the code is indeed placed in the public domain. Enjoy!

Wednesday, July 18, 2012

Enforcing concrete class in Python

Interface-based programming is one of the two principles that Gang of Four book advocates for. This style of programming makes the contract between a provider and a consumer crystal clear while securely hides the implementation details and allows for easy modification.

Hence, we can find the concepts of interface and abstract class in virtually all object oriented programming languages. An interface provides a pure contract without any implementation. A concrete class implements all of them. And an abstract class lies somewhere in between with some unimplemented contracts, and some implemented ones.

Python, in its pureness, does not facilitate this separation of concerns. There is no interface nor abstract class in Python. And the usual way to define an interface or abstract class in Python is to raise NotImplementedError in the methods which subclasses need to implement.

Even though it usually works fine in practice, this does not enforce a concrete subclass to implement all unimplemented methods. When a concrete subclass forgets to implement some methods, the Python interpreter still happily runs the code until it hits the raise statement.

So I wrote a quick decorator to ensure that if a class is marked concrete then all its functions must not raise NotImplementedError. A SyntaxError would be thrown if any method (even inherited one) was left unimplemented. The check happens at class definition time.

To be clear, Python does have Abstract Base Class (ABC) but that is different from the regular interface/abstract class concept.

'''Set of (class) decorators to help with interface programming.


    class subclass(parent):

Released to the public domain.

Nam T. Nguyen, 2012


import dis
import types
import unittest

def concrete(orig_class):
    '''A decorator to ensure that a concrete class has no un-implemented

    An un-implemented method is defined similarly to::

        def method(...):
            raise NotImplementedError()

    This, when translated to bytecode, looks like::

        LOAD_GLOBAL       0 (NotImplementedError)
        CALL_FUNCTION     1
        RAISE_VARARGS     1

    Or when ``raise NotImplementedError``::

        LOAD_GLOBAL       0 (NotImplementedError)
        RAISE_VARARGS     1

    The check here is for such pattern.


    for name in dir(orig_class):
        func = getattr(orig_class, name)
        # correct type?
        if type(func) not in (types.FunctionType, types.MethodType):
        # check if first name is NotImplementedError
        if len(func.func_code.co_names) < 1 or \
                func.func_code.co_names[0] != 'NotImplementedError':
        # and RAISE_VARARGS somewhere after that
        for position in (3, 6):
            if len(func.func_code.co_code) < position:
            opcode = ord(func.func_code.co_code[position])
            if dis.opname[opcode] == 'RAISE_VARARGS':
                raise SyntaxError('Function %s.%s must be implemented.' % (
                        orig_class.__name__, func.func_name))

    return orig_class

class ConcreteTest(unittest.TestCase):

    def test_raise(self):
        class test1:
            def method(self):
                raise NotImplementedError
        self.assertRaises(SyntaxError, concrete, test1)

        class test2:
            def method(self):
                raise NotImplementedError()
        self.assertRaises(SyntaxError, concrete, test2)

    def test_not_raise(self):
        class test:
            def method(self):
                return NotImplementedError()

    def test_subclass(self):
        class parent(object):
            def override_me(self):
                raise NotImplementedError()
        class test(parent):
            def leave_it(self):
        self.assertRaises(SyntaxError, concrete, test)

if __name__ == '__main__':