Friday, June 01, 2018

Scala Tips - separating common from varying

It is a best practice to separating common from varying in the Scala. Common parts are implemented as functions and the varying parts will be arguments:

def createSeqOnCriteria(start:Int, end:Int,                              // createSeqOnCriteria: createSeqOnCriteria[](val start: Int,val end: Int,val criteria: Int => Boolean) => scala.collection.immutable.IndexedSeq[Int]                    
                        criteria: Int => Boolean) = {                    //                                                                                                                                                                       
  for (i <- start to end if criteria(i)) yield i                         //                                                                                                                                                                       
}                                                                        //                                                                                                                                                                       
                                                                         //                                                                                                                                                                       
// you can pass any functional literal Int => Boolean                    //                                                                                                                                                                       
createSeqOnCriteria(1,10, { a:Int => a > 5})                             // res0: scala.collection.immutable.IndexedSeq[Int] = Vector(6, 7, 8, 9, 10)                                                                                             
createSeqOnCriteria(1,10, { a:Int => a > 5 && a <8})                     // res1: scala.collection.immutable.IndexedSeq[Int] = Vector(6, 7)                                                                                                       
                                                                         //                                                                                                                                                                       
// using placeholders with only bound variables                          //                                                                                                                                                                       
def createSeqOnCriteria1(start:Int, end:Int, data:Int,                   // createSeqOnCriteria1: createSeqOnCriteria1[](val start: Int,val end: Int,val data: Int,val criteria: (Int, Int) => Int) => scala.collection.immutable.IndexedSeq[Int] 
                        criteria: (Int,Int) => Int) = {                  //                                                                                                                                                                       
  for (i <- start to end) yield criteria(i,data)                         //                                                                                                                                                                       
}                                                                        //                                                                                                                                                                       
                                                                         //                                                                                                                                                                       
createSeqOnCriteria1(1,10, 5, _.max(_))                                  // res2: scala.collection.immutable.IndexedSeq[Int] = Vector(5, 5, 5, 5, 5, 6, 7, 8, 9, 10)                                                                              
createSeqOnCriteria1(1,10, 5, _.min(_))                                  // res3: scala.collection.immutable.IndexedSeq[Int] = Vector(1, 2, 3, 4, 5, 5, 5, 5, 5, 5)                                                                               
createSeqOnCriteria1(1,10, 5, _ * _)                                     // res4: scala.collection.immutable.IndexedSeq[Int] = Vector(5, 10, 15, 20, 25, 30, 35, 40, 45, 50)                                                                      
                                                                         //                                                                                                                                                                       
// simplified version                                                    //                                                                                                                                                                       
def createSeqOnCriteria2(start:Int, end:Int, criteria: (Int) => Int) = { // createSeqOnCriteria2: createSeqOnCriteria2[](val start: Int,val end: Int,val criteria: Int => Int) => scala.collection.immutable.IndexedSeq[Int]                      
  for (i <- start to end) yield criteria(i)                              //                                                                                                                                                                       
}                                                                        //                                                                                                                                                                       
                                                                         //                                                                                                                                                                       
//simplified version with the free variable                              //                                                                                                                                                                       
var n = 4                                                                // n: Int = 4                                                                                                                                                            
createSeqOnCriteria2(1,10, _ * n)                                        // res5: scala.collection.immutable.IndexedSeq[Int] = Vector(4, 8, 12, 16, 20, 24, 28, 32, 36, 40)                                                                       

Above examples shows few examples how to you placeholder (_) as well 1. Going bit further, high order functions take functions as argument parameters.

Process of currying

Here the concept of currying in the Scala 2. In the following code, first illustrate the functional literal to form a partial function. Second is the the use of curried function directly. The third approach is partial function with the place holder.

// Illustrate the curring process               //                                                         
def add (a:Int): Int => Int = (b:Int) => a + b  // add: add[](val a: Int) => Int => Int                    
val partA = add(1)                              // partA: Int => Int = <function1>                         
val partB = partA(2)                            // partB: Int = 3                                          
                                                //                                                         
// The curried function is as follows           //                                                         
def curriedAdd(a:Int)(b:Int) = a +b             // curriedAdd: curriedAdd[](val a: Int)(val b: Int) => Int 
curriedAdd(1)(2)                                // res0: Int = 3                                           
                                                //                                                         
// As partial function                          //                                                         
val curriedPartA = curriedAdd(1)_               // curriedPartA: Int => Int = <function1>                  
curriedPartA(2)                                 // res1: Int = 3                                           

As explained in the Programming in Scala section 9.4 2, if a control pattern repeated in multiple parts of the code, better consider to create new control structure for that.

Reference


  1. Blog post Scala Tips: Functions and Closures 

  2. Programming in Scala, Third Edition, by Bill Venners, Lex Spoon, Martin Odersky, Publisher: Artima Press, Release Date: April 2016, ISBN: 9780981531687. 

Thursday, May 31, 2018

Scala Tips: Functions and Closures

In the Scala functions are first class. For example function literal can be possible to assign to object as follows:

val f = (x:Int ) => x + x //f: Int => Int = <function1>
f(2) // res0: Int = 4

The different between function literal and function value is that literal exists in source code and the value exists in the runtime. Compiler do the target typing if you don't specify the type for the x in the above source. To more precise, you can use the _ as a placeholder instead of x bound variable in the above code because x is a parameter to the function. For simplicity :

//collection supports
val l1 = for (i <- 1 to 10) yield i //l1: scala.collection.immutable.IndexedSeq[Int] = Vector(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
l1.filter(_ > 3 ) //res1: scala.collection.immutable.IndexedSeq[Int] = Vector(4, 5, 6, 7, 8, 9, 10)

In the above source, when _ used that is called partially applied function because don't supply all of the arguments needed by the expression in the function.

def sum(a:Int, b:Int, c:Int) = (a + b) * c                // sum: sum[](val a: Int,val b: Int,val c: Int) => Int 
val a = sum _ // leave off all the parameters             // a: (Int, Int, Int) => Int = <function3>             
val multiply = a(1 , 2, _:Int) // leave off one parameter // multiply: Int => Int = <function1>                  
multiply(3)                                               // res0: Int = 9                                       

In the above source, sum is a closed term because there is only bound variables need to satisfy the functional literal.

But if there is free variable which is not supplied via bound variable parameters, that is called open term.

                                                                                            //                                                                                                                 
var noOfAccess : Int = 0                                                                    // noOfAccess: Int = 0                                                                                             
val c = scala.collection.mutable.Map("SL"->"Sri Lanka", "AU"-> "Australia")                 // c: scala.collection.mutable.Map[String,String] = Map(SL -> Sri Lanka, AU -> Australia)                          
def findCountry(countries: scala.collection.mutable.Map[String, String]) ={                 // findCountry: findCountry[](val countries: scala.collection.mutable.Map[String,String]) => String => String      
  (code:String) => {noOfAccess += 1; countries(code)}                                       //                                                                                                                 
}                                                                                           //                                                                                                                 
                                                                                            //                                                                                                                 
val closure = findCountry(c)                                                                // closure: String => String = <function1>                                                                         
closure("AU")                                                                               // res0: String = Australia                                                                                        
                                                                                            //                                                                                                                 
                                                                                            //                                                                                                                 
//if change the free variable                                                               //                                                                                                                 
c += "UK"-> "United Kingdom"                                                                // res1: scala.collection.mutable.Map[String,String] = Map(SL -> Sri Lanka, AU -> Australia, UK -> United Kingdom) 
closure("UK")                                                                               // res2: String = United Kingdom                                                                                   
                                                                                            //                                                                                                                 
print(noOfAccess)                                                                           // 2res3: Unit = ()                                                                                                
                                                                                            //                                                                                                                 
val closure1 = findCountry(c)                                                               // closure1: String => String = <function1>                                                                        
closure("UK")                                                                               // res4: String = United Kingdom                                                                                   
print(noOfAccess)                                                                           // 3res5: Unit = ()                                                                                                
                                                                                            //                                                                                                                 

In the above source, closure find the open term which need to be closed. The varaible countries is a free variable (is not a parameter from the functional literal but) within the scope where functional literal is defined. The variable closure has reference to the countries. If you change the free variable still the closure adjust for that. Important to note that free variable changed outside of the closure. Also closure can change the free variable as well. As shown in the above example, same noOfAccess visible to the clousre and closure1.

Partial Functions

Here the example for partial function

// partial functions                               //                                                         
case class Human(name:String, gender:Boolean)      // defined class Human                                     
val female:Human = Human("Theresa", true)          // female: Human = Human(Theresa,true)                     
val male: Human = Human("Mike", false)             // male: Human = Human(Mike,false)                         
                                                   //                                                         
val pregnant: PartialFunction[Human,String] = {    // pregnant: PartialFunction[Human,String] = <function1>   
  case Human(_, true) => "Lady"                    //                                                         
}                                                  //                                                         
                                                   //                                                         
val notPregnant: PartialFunction[Human,String] = { // notPregnant: PartialFunction[Human,String] = <function1>
  case Human(_, false) => "Man"                    //                                                         
}                                                  //                                                         
                                                   //                                                         
val person = pregnant orElse notPregnant           // person: PartialFunction[Human,String] = <function1>     
                                                   //                                                         
pregnant.isDefinedAt(female)                       // res6: Boolean = true                                    
pregnant.isDefinedAt(male)                         // res7: Boolean = false                                   
                                                   //                                                         
person(female)                                     // res8: String = Lady                                     
person(male)                                       // res9: String = Man                                      
                                                   //                                                         
pregnant                                           // res10: PartialFunction[Human,String] = <function1>      

It is important to note that isDefined function which test the applicability of the class to the partial function.

Friday, May 25, 2018

Python Mocking Examples

Here the first example using decorator in python 2.7

import unittest
import random
import mock

def myrandom(p):
    return random.random() > p

class Test(unittest.TestCase):

    @mock.patch('random.random')
    def test_myrandom(self, mock_random):
        mock_random.return_value = 0.1
        val = myrandom(0.0)
        assert val  > 0
        assert mock_random.call_count == 1

if __name__ == '__main__':
    unittest.main()

Here the example for assert_callled_with() function

import unittest
import mock
import example

class Test(unittest.TestCase):

    @mock.patch('example.hello')
    def test1(self,mock_hello):
        x = 'Oj'
        example.hello(x)  # Uses patched example.func
        mock_hello.assert_called_with(x)

if __name__ == '__main__':
    unittest.main()

Above test can be ran using context manager:

import unittest
import mock
import example

class Test(unittest.TestCase):

    def test1(self):
        x = "OJ"
        with mock.patch('example.hello') as mock_hello:
            example.hello(x)
            mock_hello.assert_called_with(x)

if __name__ == '__main__':
    unittest.main()

If you are testing the method in the same module:

import unittest
import mock

def hello(name):
    return '{}'.formt(name)

class Test(unittest.TestCase):

    def test1(self):
        x = "OJ"
        with mock.patch('__main__.hello') as mock_hello:
            hello(x)
            mock_hello.assert_called_with(x)

if __name__ == '__main__':
    unittest.main()

This is how return value is working

import unittest
import mock

def hello(name):
    return None

class Test(unittest.TestCase):

    @mock.patch('__main__.hello', return_value='Oj')
    def test1(self,mock_hello):
        x = 'Oj'
        assert mock_hello(x) == x


if __name__ == '__main__':
    unittest.main()

If you run in the context manager:

import unittest
import mock

def hello(name):
    return None

class Test(unittest.TestCase):

    def test1(self):
        x = "OJ"
        with mock.patch('__main__.hello', return_value='Hello {}'.format(x)) as mock_hello:
            assert hello(x) == 'Hello {}'.format(x)

if __name__ == '__main__':
    unittest.main()

How to mock a method

import unittest
import mock

class MyMock(object):
    def __init__(self,name):
        self.name = name

attrs = {'method.return_value':MyMock('Tixtrac'), 'other.side_effect':KeyError}
mock = mock.Mock(some_attribute = 'eggs', **attrs)
mock.some_attribute
obj = mock.method()
print(obj.name)
mock.other()

In the above source, the method has been mocked.

Mock the class method, here because spec=True , you can only mock the methods belongs to SomeClass.

import unittest
import mock

class SomeClass(object):
    def hello():
        return 'Hello'

class Test(unittest.TestCase):
    @mock.patch('__main__.SomeClass', spec=True)
    def test_SomeClass(self, mc):

        #mock the method
        instance = mc.return_value
        instance.hello.return_value = 'Foo'
        #instance.m.return_value = 'Foo' #failed because spec=True

        #create instance of SomeClass
        sc = SomeClass()

        assert sc is instance
        assert sc.hello() == 'Foo'
        # assert sc.m() == 'Foo' #faiedl because spec=True

if __name__ == '__main__':
    unittest.main()

Here the way to mock the property decorator:

import unittest
import mock

class Foo(object):
    @property
    def foo(self):
        return 'foo thing'
    @foo.setter
    def foo(self, value):
        pass

class Test(unittest.TestCase):

    @mock.patch('__main__.Foo.foo', new_callable=mock.PropertyMock)
    def test1(self,mock):
        mock.return_value = 'test'
        foo = Foo()
        print('val: {}'.format(foo.foo))


if __name__ == '__main__':
    unittest.main()

Use alternative class for the default MagicMock using new_callable:

import unittest
import mock
from StringIO import StringIO
def foo():
    print 'Hello'

class Test(unittest.TestCase):
    @mock.patch('sys.stdout', new_callable=StringIO)
    def test_foo(self, mock): #mocked the stdout here
        foo() #first call the method
        assert mock.getvalue() == 'Hello\n'

if __name__ == '__main__':
    unittest.main()

Patch object

This can be used as:

  • decorator
  • class decorator
  • context manager

Patch object is either two or three arguments.

  1. object to be patched
  2. attribute name
  3. object to replace the attribute with

Here the example:

import unittest
import mock


class SomeClass(object):
    def hello(name):
        return '{} world!'.format(name)


class Test(unittest.TestCase):
    @mock.patch.object(SomeClass, 'hello')
    def test_SomeClass(self, mc):
        hello_str = 'Oj'
        SomeClass.hello(hello_str)
        mc.assert_called_with(hello_str)

if __name__ == '__main__':
    unittest.main()

Mocking the dictionary:

import unittest
import mock
import os

class Test(unittest.TestCase):

    @mock.patch.dict(os.environ, {'TARGET_STREAM' : 'test'})
    def test_dict(self):
        assert os.environ['TARGET_STREAM'] == 'test'

if __name__ == '__main__':
    unittest.main()

For the existing dictionary:

import unittest
import mock
import os
foo = {}
class Test(unittest.TestCase):

    @mock.patch.dict(foo, {'TARGET_STREAM' : 'test'})
    def test_dict(self):
        assert foo['TARGET_STREAM'] == 'test'

if __name__ == '__main__':
    unittest.main()

Mock the module using dictionary

import unittest
import mock
import os

mymodule = mock.MagicMock()
mymodule.hello.return_value = 'Hello'
class Test(unittest.TestCase):

    @mock.patch.dict('sys.modules', mymodule=mymodule)
    def test_dict(self):
        import mymodule #import this to test
        assert mymodule.hello('2') == 'Hello'

if __name__ == '__main__':
    unittest.main()

Tips to Mock AWS Boto 3

First create the sharerable context:

from contextlib import contextmanager
import os

@contextmanager
def lambdaEnv(stream_name):
    STREAM_NAME = stream_name
    os.environ['LOGGING_LEVEL'] = 'DEBUG'
    os.environ['SOURCE_STREAM'] = STREAM_NAME
    os.environ['TARGET_BUCKET'] = 'ojithak'
    os.environ['TARGET_BUCKET_ROOT_DIR'] = 'ticketingkin'

    os.environ['KS_CUSTOMER_SALES'] = 'arn:aws:kinesis:ap-southeast-2:167856709666:stream/customersales_in'#only use this
    os.environ['RT_SALES'] = 'sales'
    os.environ['KS_STORM'] = 'arn:aws:kinesis:ap-southeast-2:167856709666:stream/storm-dev'
    os.environ['RT_PAYMENT_CT'] = 'Sales.cdc.dbo_Payment_CT'
    os.environ['RT_ORDERLINES'] = 'OrdersQA.Orderlines'
    os.environ['RT_ORDERLINE_PAYMENTS'] = 'OrdersQA.Orderlines'
    os.environ['KS_TICKETING'] = 'arn:aws:kinesis:ap-southeast-2:167856709666:stream/ticketing-dev'
    os.environ['KS_ORDERS'] = 'arn:aws:kinesis:ap-southeast-2:167856709666:stream/new-orders-dev'
    os.environ['KS_TIXTRACK'] = 'arn:aws:kinesis:ap-southeast-2:167856709666:stream/tixtrack_dev'

    logging.basicConfig(format='[%(asctime)s] p%(process)s {%(filename)s:%(funcName)s:%(lineno)d} %(levelname)s - %(message)s',
                        datefmt='%d-%m-%Y:%H:%M:%S',
                        level=os.environ['LOGGING_LEVEL'])
    ch = logging.StreamHandler()
    log = logging.getLogger()
    log.addHandler(ch)
    log.info("logger is ready...")
    yield log

Here the test case setup:

from mock import Mock
from test_common import lambdaEnv


STREAM_NAME = 'stream-dev'



class TestTicketingKinesisToS3(unittest.TestCase):
    kinesis_obj = boto3.client
    event = None

    def setUp(self):
        unittest.TestCase.setUp(self)

        data1 = '{"RecordType":"Transaction","...}'

        data2 = '{"RecordType":"Transaction","...}'

        tt = [{u'eventID': u'shardId-000000000000:49574118225392902538999142319068611050512700818631688194',
               u"eventSource": u"aws:kinesis",
               u"eventVersion": u"1.0",
               u"invokeIdentityArn": u"arn:aws:iam::167856709666:role/lambda-kinesis-execution-role",
               u"eventName": u"aws:kinesis:record",
               u"eventSourceARN": u"arn:aws:kinesis:ap-southeast-2:167856709666:stream/ticketing-dev",
               u'kinesis': {
                   u'approximateArrivalTimestamp': time.time(),
                   u'data': base64.b64encode(utils.gzipDataBlob(data1)),
                   u'partitionKey': u'partitionKey-0',
                   u'sequenceNumber': u'49574118225392902538999142319068611050512700818631688194'}},
              {u'eventID': u'shardId-000000000000:49574118225392902538999142319069819976332315447806394370',
               u"eventSource": u"aws:kinesis",
               u"eventVersion": u"1.0",
               u"invokeIdentityArn": u"arn:aws:iam::167856709666:role/lambda-kinesis-execution-role",
               u"eventName": u"aws:kinesis:record",
               u"eventSourceARN": u"arn:aws:kinesis:ap-southeast-2:167856709666:stream/ticketing-dev",
               u'kinesis': {
                   u'approximateArrivalTimestamp': time.time(),
                   u'data': base64.b64encode(utils.gzipDataBlob(data2)),
                   u'partitionKey': u'partitionKey-1',
                   u'sequenceNumber': u'49574118225392902538999142319069819976332315447806394370'}},
              {u'eventID': u'shardId-000000000000:49569562892134134084378247283249679591017585731862265858',
               u"eventSource": u"aws:kinesis",
               u"eventVersion": u"1.0",
               u"invokeIdentityArn": u"arn:aws:iam::167856709666:role/lambda-kinesis-execution-role",
               u"eventName": u"aws:kinesis:record",
               u"eventSourceARN": u"arn:aws:kinesis:ap-southeast-2:167856709666:stream/ticketing-dev",
               u'kinesis': {
                   u'approximateArrivalTimestamp': time.mktime(datetime.datetime(2017, 6, 14, 16, 33, 24, 775000,
                                                                                 tzinfo=pytz.timezone(
                                                                                     'Australia/Sydney')).timetuple()),
                   u'data': base64.b64encode(
                       '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\...'),
                   u'PartitionKey': u'mBuDKy',
                   u'sequenceNumber': u'49569562892134134084378247283249679591017585731862265858'}}
              ]

        self.event = {'Records': tt}

Now the test case:

    # @mock.patch('boto3.client', side_effect=kinesisClient)
    # def test_handler(self,mock_func):
    def test_handler(self):
        with lambdaEnv(STREAM_NAME) as log:
            import KinesisToS3
            log.debug("create context for lambda function")
            context = Mock()
            context.function_name = 'test'
            context.aws_request_id = 'd75ec968-30b4-11e8-bc5a-79557d1008ed'
            KinesisToS3.handler(self.event, context)
        # assert mock_func.called


if __name__ == '__main__':
    unittest.main()