url routing, argument extraction and conversion, url caching

README.md

-REST micro-framework for Twisted
+Twisted REST micro-framework
+================================
 
-Based on Flask API, with integrated multiprocessing support
-for full usage of all CPUs
\ No newline at end of file
+Based on *Flask* API, with integrated multiprocessing support for full usage of all CPUs. 
+Provides a more Flask/Sinatra-style API on top of the core *twisted.web* APIs.
+
+Geared towards creating REST-oriented server platforms (e.g. as a source of data for a Javascript MVC app).
+Tested exclusively on PyPy for maximum performance.
+
+Example:
+^^^^^^^
+
+	from corepost.server import CorePost
+	from corepost.enums import Http
+	
+	app = CorePost()
+	
+	@app.route("/",Http.GET)
+	def root(request):
+	    return request.path
+	
+	@app.route("/test",Http.GET)
+	def test(request):
+	    return request.path
+	
+	@app.route("/test/<int:numericid>/test2/<stringid>",Http.GET)
+	def test_get_resources(request,numericid,stringid,**kwargs):
+	    return "%s - %s" % (numericid,stringid)
+	
+	if __name__ == '__main__':
+	    app.run()
+	    
+Performance
+^^^^^^^^^^^
+
+Pushing 8,000+ TPS on a simple 'Hello World' app using 'ab -n 100000 -c 200' 
+for benchmarking while running on PyPy 1.6
+
+Plans
+^^^^^
+
+* match all the relevant features of the Flask API
+* integrate twisted.internet.processes in order to scale to multiple CPU cores : http://pypi.python.org/pypi/twisted.internet.processes
\ No newline at end of file

src/corepost/enums.py

+'''
+Common enums
+
+@author: jacekf
+'''
+
+class Http:
+    """Enumerates HTTP methods"""
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    DELETE = "DELETE"
+
+class MediaType:
+    """Enumerates media types"""    
+    WILDCARD = "*/*"
+    APPLICATION_XML = "application/xml"
+    APPLICATION_ATOM_XML = "application/atom+xml"
+    APPLICATION_XHTML_XML = "application/xhtml+xml"
+    APPLICATION_SVG_XML = "application/svg+xml"
+    APPLICATION_JSON = "application/json"
+    APPLICATION_FORM_URLENCODED = "application/x-www-form-urlencoded"
+    MULTIPART_FORM_DATA = "multipart/form-data"
+    APPLICATION_OCTET_STREAM = "application/octet-stream"
+    TEXT_PLAIN = "text/plain"
+    TEXT_XML = "text/xml"
+    TEXT_HTML = "text/html"
\ No newline at end of file

src/corepost/server.py

 '''
-Created on 2011-08-23
+Main server classes
 
 @author: jacekf
 '''
@@ -8,63 +8,85 @@ from twisted.internet import reactor
 from twisted.web.resource import Resource
 from twisted.web.server import Site
 from collections import defaultdict
-
-class Http:
-    """Enumerates HTTP methods"""
-    GET = "GET"
-    POST = "POST"
-    PUT = "PUT"
-    DELETE = "DELETE"
-
-class MediaType:
-    """Enumerates media types"""    
-    WILDCARD = "*/*"
-    APPLICATION_XML = "application/xml"
-    APPLICATION_ATOM_XML = "application/atom+xml"
-    APPLICATION_XHTML_XML = "application/xhtml+xml"
-    APPLICATION_SVG_XML = "application/svg+xml"
-    APPLICATION_JSON = "application/json"
-    APPLICATION_FORM_URLENCODED = "application/x-www-form-urlencoded"
-    MULTIPART_FORM_DATA = "multipart/form-data"
-    APPLICATION_OCTET_STREAM = "application/octet-stream"
-    TEXT_PLAIN = "text/plain"
-    TEXT_XML = "text/xml"
-    TEXT_HTML = "text/html"
+from enums import MediaType
     
 class RequestRouter:
+    """ Common class for containing info related to routing a request to a function """
     
     __urlMatcher = re.compile(r"<(int|float|):?([a-zA-Z0-9]+)>")
-    __urlRegexReplace = {"":r"(.+)","int":r"(d+)","float":r"(d+\.d+)"}
+    __urlRegexReplace = {"":r"(?P<arg>.+)","int":r"(?P<arg>\d+)","float":r"(?P<arg>\d+.?\d*)"}
+    __typeConverters = {"int":int,"float":float}
     
-    """ Common class for containing info related to routing a request to a function """
-    def __init__(self,f,url,method,accepts,produces):
+    def __init__(self,f,url,method,accepts,produces,cache):
         self.__url = url
         self.__method = method
         self.__accepts = accepts
         self.__produces = produces
+        self.__cache = cache
         self.__f = f
-        self.__args = [] # dict of arg names -> group index
+        self.__argConverters = {} # dict of arg names -> group index
         
         #parse URL into regex used for matching
         m = RequestRouter.__urlMatcher.findall(url)
         
-        self.__matchUrl = url
+        self.__matchUrl = "^%s$" % url
         for match in m:
-            self.__args.append(match[1])
             if len(match[0]) == 0:
                 # string
-                self.__matchUrl = self.__matchUrl.replace("<%s>" % match[1],RequestRouter.__urlRegexReplace[match[0]])
+                self.__argConverters[match[1]] = None
+                self.__matchUrl = self.__matchUrl.replace("<%s>" % match[1],
+                                    RequestRouter.__urlRegexReplace[match[0]].replace("arg",match[1]))
             else:
                 # non string
-                self.__matchUrl = self.__matchUrl.replace("<%s:%s>" % match,RequestRouter.__urlRegexReplace[match[0]])
+                self.__argConverters[match[1]] = RequestRouter.__typeConverters[match[0]]
+                self.__matchUrl = self.__matchUrl.replace("<%s:%s>" % match,
+                                    RequestRouter.__urlRegexReplace[match[0]].replace("arg",match[1]))
 
         self.__matcher = re.compile(self.__matchUrl)
         
-    def getMatch(self,url):
-        return self.__matcher.findall(url)
+    @property
+    def cache(self):
+        """Indicates if this URL should be cached or not"""
+        return self.__cache    
+        
+    def getArguments(self,url):
+        """
+        Returns None if nothing matched (i.e. URL does not match), empty dict if no args found (i,e, static URL)
+        or dict with arg/values for dynamic URLs
+        """
+        g = self.__matcher.search(url)
+        if g != None:
+            args = g.groupdict()
+            # convert to expected datatypes
+            if len(args) > 0:
+                for name in args.keys():
+                    converter = self.__argConverters[name]
+                    if converter != None:
+                        args[name] = converter(args[name])
+            return args
+        else:
+            return None
+        
+    def call(self,request,**kwargs):
+        """Forwards call to underlying method"""
+        return self.__f(request,**kwargs)
+
+class CachedUrl():
+    """
+    Used for caching URLs that have been already routed once before. Avoids the overhead
+    of regex processing on every incoming call for commonly accessed REST URLs
+    """
+    def __init__(self,router,args):
+        self.__router = router
+        self.__args = args
         
-    def call(self,**kwargs):
-        self.__f(**kwargs)
+    @property
+    def router(self):
+        return self.__router
+    
+    @property
+    def args(self):
+        return self.__args
     
 class CorePost(Resource):
     '''
@@ -77,24 +99,24 @@ class CorePost(Resource):
         Constructor
         '''
         self.__urls = defaultdict(dict)
-        self.__cachedUrls = defaultdict(dict) # used to avoid routing request to function on every request
+        self.__cachedUrls = defaultdict(dict)
         self.__methods = {}
 
-    def __registerFunction(self,f,url,methods,accepts,produces):
+    def __registerFunction(self,f,url,methods,accepts,produces,cache):
         if f not in self.__methods.values():
             if not isinstance(methods,(list,tuple)):
                 methods = (methods,)
 
             for method in methods:
-                self.__urls[method][url] = f
-                rq = RequestRouter(f, url, method, accepts, produces)
+                rq = RequestRouter(f, url, method, accepts, produces,cache)
+                self.__urls[method][url] = rq
             
             self.__methods[url] = f
 
-    def route(self,url,methods=[],accepts=MediaType.WILDCARD,produces=None):
+    def route(self,url,methods=[],accepts=MediaType.WILDCARD,produces=None,cache=True):
         """Main decorator for registering REST functions """
         def wrap(f):
-            self.__registerFunction(f, url, methods, accepts, produces)
+            self.__registerFunction(f, url, methods, accepts, produces,cache)
             return f
         return wrap
 
@@ -115,10 +137,21 @@ class CorePost(Resource):
         return self.__renderUrl(request)
     
     def __renderUrl(self,request):
-        if request.path in self.__urls[request.method].keys():
-            return self.__urls[request.method][request.path]()
+        """Finds the appropriate router and dispatches the request to the registered function"""
+        # see if already cached
+        if request.path in self.__cachedUrls[request.method]:
+            cachedUrl = self.__cachedUrls[request.method][request.path]
+            return cachedUrl.router.call(request,**cachedUrl.args)
         else:
-            return self.__renderError(request,404,"URL '%s' not found\n" % request.path)
+            # first time this URL is called
+            for router in self.__urls[request.method].values():
+                args = router.getArguments(request.path)
+                if args != None:
+                    if router.cache:
+                        self.__cachedUrls[request.method][request.path] = CachedUrl(router, args)
+                    return router.call(request,**args)
+        
+        return self.__renderError(request,404,"URL '%s' not found\n" % request.path)
     
     def __renderError(self,request,code,message):
         """Common method for rendering errors"""

src/corepost/test/server_test.py

 '''
-Created on 2011-08-23
-
+Server tests
 @author: jacekf
 '''
 
-from corepost.server import CorePost, Http
+from corepost.server import CorePost
+from corepost.enums import Http
 
 app = CorePost()
 
 @app.route("/",Http.GET)
-def test():
-    return "test"
+def root(request):
+    return request.path
+
+@app.route("/test",Http.GET)
+def test(request):
+    return request.path
 
-@app.route("/test/<int:jacek>/test/<stringid>/float/<float:floater>/test2",(Http.POST,Http.PUT))
-def test_post():
-    return "test POST/PUT"
+@app.route("/test/<int:jacek>/yo/<someid>",Http.GET)
+def test_get_resources(request,jacek,someid,**kwargs):
+    return "%s - %s" % (jacek,someid)
 
 if __name__ == '__main__':
     app.run()
\ No newline at end of file