0.0.7: content types

README.md

@@ -107,6 +107,18 @@ Example:
 	@route("/int/<int:intarg>/float/<float:floatarg>/string/<stringarg>",Http.GET)
 	def test(self,request,intarg,floatarg,stringarg,**kwargs):
 		pass
+
+@defer.inlineCallbacks support
+------------------------------
+
+If you want a deferred async method, just use *defer.returnValue()*
+
+	@route("/",Http.GET)
+	@defer.inlineCallbacks
+	def root(self,request,**kwargs):
+		val1 = yield db.query("SELECT ....")
+		val2 = yield db.query("SELECT ....")
+		defer.returnValue(val1 + val2)
 	    
 Argument validation
 -------------------
@@ -141,18 +153,79 @@ for list of available validators:
 
 * Common <http://www.formencode.org/en/latest/modules/validators.html#module-formencode.validators>
 * National <http://www.formencode.org/en/latest/modules/national.html#module-formencode.national>
-	    
-@defer.inlineCallbacks support
-------------------------------
 
-If you want a deferred async method, just complete the request yourself, instead of returning a string response
+Content types
+-------------
 
-	@route("/",Http.GET)
-	@defer.inlineCallbacks
-	def root(self,request,**kwargs):
-		val = yield db.query("SELECT ....")
-		request.write(val)
-		request.finish()
+CorePost integrates support for JSON, YAML and XML (partially) based on request content types.
+
+Parsing of incoming content
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Based on the incoming content type in POST/PUT requests,
+the body will be automatically parsed to JSON, YAML and XML (ElementTree) and attached to the request:
+
+* request.json
+* request.yaml
+* request.xml
+
+    @route("/post/json",(Http.POST,Http.PUT))
+    def test_json(self,request,**kwargs):
+        return "%s" % json.dumps(request.json)
+
+    @route("/post/xml",(Http.POST,Http.PUT))
+    def test_xml(self,request,**kwargs):
+        return "%s" % ElementTree.tostring(request.xml)
+
+    @route("/post/yaml",(Http.POST,Http.PUT))
+    def test_yaml(self,request,**kwargs):
+        return "%s" % yaml.dump(request.yaml)
+
+
+Routing requests by incoming content type
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Based on the incoming content type in POST/PUT requests,
+the *same* URL can be hooked up to different router methods:
+
+    @route("/post/by/content",(Http.POST,Http.PUT),MediaType.APPLICATION_JSON)
+    def test_content_app_json(self,request,**kwargs):
+        return request.received_headers[HttpHeader.CONTENT_TYPE]
+
+    @route("/post/by/content",(Http.POST,Http.PUT),(MediaType.TEXT_XML,MediaType.APPLICATION_XML))
+    def test_content_xml(self,request,**kwargs):
+        return request.received_headers[HttpHeader.CONTENT_TYPE]
+
+    @route("/post/by/content",(Http.POST,Http.PUT),MediaType.TEXT_YAML)
+    def test_content_yaml(self,request,**kwargs):
+        return request.received_headers[HttpHeader.CONTENT_TYPE]
+
+    @route("/post/by/content",(Http.POST,Http.PUT))
+    def test_content_catch_all(self,request,**kwargs):
+        return MediaType.WILDCARD
+
+Converting Python objects to content type based on what caller can accept
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Instead of returning string responses, the code can just return Python objects.
+Depending whether the caller can accept JSON (default) or YAML, the Python objects	    
+will be automatically converted:
+
+    @route("/return/by/accept")
+    def test_return_content_by_accepts(self,request,**kwargs):
+        val = [{"test1":"Test1"},{"test2":"Test2"}]
+        return val
+        
+Calling this URL with "Accept: application/json" will return:
+
+	[{"test1": "Test1"}, {"test2": "Test2"}]
+	
+Calling it with "Accept: text/yaml" will return:
+
+	- {test1: Test1}
+	- {test2: Test2} 
+
+*Note*: marshalling to XML will be supported in a future release. There is no default Python library that does this automatically.
 
 HTTP codes
 ------------------

corepost/enums.py

@@ -14,6 +14,7 @@ class Http:
 class HttpHeader:
     """Enumerates common HTTP headers"""
     CONTENT_TYPE = "content-type"
+    ACCEPT = "accept"
 
 class MediaType:
     """Enumerates media types"""    

corepost/test/feature/content_types.feature

@@ -172,4 +172,42 @@ total: 4443.52
 			| PUT		| XML		| <test>1</test>	| text/xml			 	| 200	|
 			| PUT		| XML		| <test>1</test>	| application/xml	 	| 200	|
 			| PUT		| YAML		| test: 2			| text/yaml			 	| 200	|
+			
+	@json @yaml @xml @return_accept
+	Scenario Outline: Return content type based on caller's Accept
+		When I prepare HTTP header 'Accept' = '<accept>'
+		When as user 'None:None' I GET 'http://127.0.0.1:8080/return/by/accept'
+		Then I expect HTTP code <code>
+		And I expect content contains '<content>'
+		
+		Examples:
+			| content													| accept				| code 	|  
+			| [{"test1": "Test1"}, {"test2": "Test2"}]					| application/json	 	| 200	|
+			| Unable to convert String response to XML automatically	| application/xml	 	| 500	| # not supported yet
+			| - {test1: Test1}\n- {test2: Test2}						| text/yaml				| 200	|
+			
+	@json @yaml @xml @return_accept_deferred
+	Scenario Outline: Return content type based on caller's Accept from Deferred methods
+		When I prepare HTTP header 'Accept' = '<accept>'
+		When as user 'None:None' I GET 'http://127.0.0.1:8080/return/by/accept/deferred'
+		Then I expect HTTP code <code>
+		And I expect content contains '<content>'
+		
+		Examples:
+			| content													| accept				| code 	|  
+			| [{"test1": "Test1"}, {"test2": "Test2"}]					| application/json	 	| 200	|
+			| Unable to convert String response to XML automatically	| application/xml	 	| 500	| # not supported yet
+			| - {test1: Test1}\n- {test2: Test2}						| text/yaml				| 200	|			
+			
+	@json @yaml @xml @return_accept
+	Scenario Outline: Return class content type based on caller's Accept
+		When I prepare HTTP header 'Accept' = '<accept>'
+		When as user 'None:None' I GET 'http://127.0.0.1:8080/return/by/accept/class'
+		Then I expect HTTP code <code>
+		And I expect content contains '<content>'
+		
+		Examples:
+			| content													| accept				| code 	|  
+			| is not JSON serializable									| application/json	 	| 500	| # not supported yet
+			| Unable to convert String response to XML automatically	| application/xml	 	| 500	| # not supported yet
 			
\ No newline at end of file

corepost/test/home_resource.py

@@ -72,6 +72,33 @@ class HomeApp(CorePost):
     @route("/post/by/content",(Http.POST,Http.PUT))
     def test_content_catch_all(self,request,**kwargs):
         return MediaType.WILDCARD
+    
+    ##################################################################
+    # one URL, serving different content types
+    ###################################################################
+    @route("/return/by/accept")
+    def test_return_content_by_accepts(self,request,**kwargs):
+        val = [{"test1":"Test1"},{"test2":"Test2"}]
+        return val
+
+    @route("/return/by/accept/deferred")
+    @defer.inlineCallbacks
+    def test_return_content_by_accept_deferred(self,request,**kwargs):
+        """Ensure support for inline callbacks and deferred"""
+        val = yield [{"test1":"Test1"},{"test2":"Test2"}]
+        defer.returnValue(val)
+
+    @route("/return/by/accept/class")
+    def test_return_class_content_by_accepts(self,request,**kwargs):
+        """Uses Python class instead of dict/list"""
+        class Test: pass
+        t1 = Test()
+        t1.test1 = "Test1"
+        t2 = Test()
+        t2.test2 = "Test2"
+        val = [t1,t2]
+        return val
+
 
 
 def run_app_home():

corepost/test/steps.py

@@ -4,7 +4,7 @@ Common Freshen BDD steps
 @author: jacekf
 '''
 from multiprocessing import Process
-import httplib2, json, re, time
+import httplib2, json, re, time, string
 from freshen import Before, Given, When, Then, scc, glc, assert_equals, assert_true #@UnresolvedImport
 from urllib import urlencode
 from corepost.test.home_resource import run_app_home
@@ -58,7 +58,7 @@ def when_as_user_i_send_get_delete_to_url(user,password,method,url):
     h = httplib2.Http()
     h.follow_redirects = False
     h.add_credentials(user, password)
-    scc.response, scc.content = h.request(url, method)
+    scc.response, scc.content = h.request(url, method, headers = scc.http_headers)
 
 @When(r"^as user '(.+):(.+)' I (POST|PUT) '(.+)' with '(.+)'\s*$")
 def when_as_user_i_send_post_put_to_url(user,password,method,url,params):
@@ -93,6 +93,12 @@ def when_i_define_http_header_with_value(header,value):
 ##################################
 # THEN
 ##################################
+def transform_content(content):
+    """Support embedded newlines"""
+    if content != None:
+        return string.replace(content,"\\n","\n")
+    else:
+        return None
 
 @Then(r"^I expect HTTP code (\d+)\s*$")
 def expect_http_code(code):
@@ -100,10 +106,12 @@ def expect_http_code(code):
 
 @Then(r"^I expect content contains '(.+)'\s*$")
 def expect_content(content):
+    content = transform_content(content)
     assert_true(scc.content.find(content) >= 0,"Did not find:\n%s\nin content:\n%s" % (content,scc.content)) 
 
 @Then(r"^I expect content contains\s*$")
 def expect_content_multiline(content):
+    content = transform_content(content)
     assert_true(scc.content.find(content) >= 0,"Did not find:\n%s\nin content:\n%s" % (content,scc.content)) 
 
 @Then(r"^I expect '([^']*)' header matches '([^']*)'\s*$")

corepost/utils.py

@@ -2,6 +2,7 @@
 Various CorePost utilities
 '''
 from inspect import getargspec
+import json
 
 def getMandatoryArgumentNames(f):
     '''Returns a tuple of the mandatory arguments required in a function'''
@@ -13,4 +14,11 @@ def getMandatoryArgumentNames(f):
     
 def getRouterKey(method,url):
     '''Returns the common key used to represent a function that a request can be routed to'''
-    return "%s %s" % (method,url)
\ No newline at end of file
+    return "%s %s" % (method,url)
+
+def convertToJson(obj):
+    """Converts to JSON, including Python classes that are not JSON serializable by default"""
+    try:
+        return json.dumps(obj)
+    except Exception as ex:
+        raise RuntimeError(str(ex))

corepost/web.py

@@ -5,7 +5,7 @@ Main server classes
 '''
 from collections import defaultdict
 from corepost.enums import Http, HttpHeader
-from corepost.utils import getMandatoryArgumentNames
+from corepost.utils import getMandatoryArgumentNames, convertToJson
 from enums import MediaType
 from formencode import FancyValidator, Invalid
 from twisted.internet import reactor, defer
@@ -14,8 +14,7 @@ from twisted.web.resource import Resource
 from twisted.web.server import Site, NOT_DONE_YET
 import re, copy, exceptions, json, yaml
 from xml.etree import ElementTree
-
-
+from xml.etree.ElementTree import Element
 
 class RequestRouter:
     ''' Common class for containing info related to routing a request to a function '''
@@ -220,12 +219,12 @@ class CorePost(Resource):
                 try:
                     # if POST/PUT, check if we need to automatically parse JSON
                     self.__parseRequestData(request)
-       
                     val = urlrouter.call(self,request,**allargs)
                 
                     #handle Deferreds natively
                     if isinstance(val,defer.Deferred):
-                        # we assume the method will call request.finish()
+                        # add callback to finish the request
+                        val.addCallback(self.__finishDeferred,request)
                         return NOT_DONE_YET
                     else:
                         #special logic for POST to return 201 (created)
@@ -236,7 +235,8 @@ class CorePost(Resource):
                             else:
                                 request.setResponseCode(201)
                         
-                        return val
+                        return self.__renderResponse(request, val)
+                    
                 except exceptions.TypeError as ex:
                     return self.__renderError(request,400,"%s" % ex)
                 except Exception as ex:
@@ -248,6 +248,56 @@ class CorePost(Resource):
         except Exception as ex:
             return self.__renderError(request,500,"Internal server error: %s" % ex)
     
+    def __renderResponse(self,request,response):
+        """
+        Takes care of automatically rendering the response and converting it to appropriate format (text,XML,JSON,YAML)
+        depending on what the caller can accept
+        """
+        if isinstance(response, str):
+            return response
+        elif isinstance(response, Response):
+            # TODO
+            return "TODO: Response"
+        else:
+            return self.__convertObjectToContentType(request, response)
+
+    def __convertObjectToContentType(self,request,obj):
+        """Takes care of converting an object (non-String) response to the appropriate format, based on the what the caller can accept"""
+        if HttpHeader.ACCEPT in request.received_headers:
+            accept = request.received_headers[HttpHeader.ACCEPT]
+            if MediaType.APPLICATION_JSON in accept:
+                request.headers[HttpHeader.CONTENT_TYPE] = MediaType.APPLICATION_JSON
+                return convertToJson(obj)
+            elif MediaType.TEXT_YAML in accept:
+                request.headers[HttpHeader.CONTENT_TYPE] = MediaType.TEXT_YAML
+                return yaml.dump(obj)
+            elif MediaType.APPLICATION_XML in accept or MediaType.TEXT_XML in accept:
+                if isinstance(obj,Element):
+                    request.headers[HttpHeader.CONTENT_TYPE] = MediaType.APPLICATION_XML
+                    return ElementTree.tostring(obj, encoding='utf-8')
+                else:
+                    raise RuntimeError("Unable to convert String response to XML automatically")
+            else:
+                # no idea, let's do JSON
+                request.headers[HttpHeader.CONTENT_TYPE] = MediaType.APPLICATION_JSON
+                return convertToJson(obj)
+        else:
+            # called has no accept header, let's default to JSON
+            request.headers[HttpHeader.CONTENT_TYPE] = MediaType.APPLICATION_JSON
+            return convertToJson(obj)
+
+    def __finishDeferred(self,val,request):
+        """Finishes any Defered/inlineCallback methods"""
+        if not request.finished:
+            if val != None:
+                try:
+                    request.write(self.__renderResponse(request,val))
+                except Exception as ex:
+                    msg = "Unexpected server error: %s\n%s" % (type(ex),ex)
+                    self.__renderError(request, 500, msg)
+                    request.write(msg)
+            request.finish()
+    
     def __renderError(self,request,code,message):
         """Common method for rendering errors"""
         request.setResponseCode(code)
@@ -279,7 +329,15 @@ class CorePost(Resource):
         factory = Site(self)
         reactor.listenTCP(port, factory)    #@UndefinedVariable
         reactor.run()                       #@UndefinedVariable
-        
+
+class Response:
+    """
+    Custom response object, can be returned instead of raw string response
+    """
+    def __init__(self,code=200,entity=None,headers={}):
+        self.code = 200
+        self.entity=entity
+        self.headers=headers         
 
 ##################################################################################################
 #

setup.py

@@ -41,6 +41,10 @@ Links
 Changelog
 `````````
 
+* 0.0.7 - automatic parsing of incoming content (JSON, YAML, XML)
+        - routing by incoming content type
+        - automatic response conversion based on caller's Accept header (JSON/YAML)
+        - support for defer.returnValue() in @inlineCallbacks route methods
 * 0.0.6 - redesigned API around classes and methods, rather than functions and global objects (after feedback from Twisted devs)
 * 0.0.5 - added FormEncode validation for arguments
 * 0.0.4 - path argument extraction, mandatory argument error checking
@@ -59,7 +63,7 @@ def read(fname):
 
 setup(
     name="CorePost",
-    version="0.0.6",
+    version="0.0.7",
     author="Jacek Furmankiewicz",
     author_email="jacekeadE99@gmail.com",
     description=("A Twisted Web REST micro-framework"),