Examples¶
Foundation¶
Web scraping¶
The user agent requests the HTML specification, and the lazy chain of
iterate of its DOM prints the rows of the table of the named character
references one by one:
import Pyjo.UserAgent from Pyjo.String.Unicode import u # Extract named character references from HTML spec tx = Pyjo.UserAgent.new().get('https://html.spec.whatwg.org') for n in tx.res.dom().iterate('#named-character-references-table tbody > tr'): u(n.at('td > code').text + ' ' + n.children('td')[1].text).trim().say()
URL manipulation¶
import Pyjo.URL # 'ssh+git://git@github.com/pyjoyment/Pyjoyment.git' url = Pyjo.URL.new('https://github.com/pyjoyment/Pyjoyment') print(url.set(scheme='ssh+git', userinfo='git', path=f'{url.path}.git').to_unsafe_str()) # 'https://pypi.org/search/?q=Pyjoyment&o=&c=Programming+Language+%3A%3A+Python' url = Pyjo.URL.new('https://pypi.org/search/') print(url.set(query=[('q', 'Pyjoyment'), ('o', ''), ('c', 'Programming Language :: Python')]))
Non-blocking TCP client/server¶
import Pyjo.IOLoop # Listen on port 3000 @Pyjo.IOLoop.server(port=3000) def server(loop, stream, cid): @stream.on def read(stream, chunk): # Process input chunk print(f'Server: {chunk.decode("utf-8")}') # Write response stream.write(b'HTTP/1.1 200 OK\x0d\x0a\x0d\x0a') # Disconnect client stream.close_gracefully() # Connect to port 3000 @Pyjo.IOLoop.client(port=3000) def client(loop, err, stream): @stream.on def read(stream, chunk): # Process input print(f'Client: {chunk.decode("utf-8")}') # Write request stream.write(b'GET / HTTP/1.1\x0d\x0a\x0d\x0a') # Add a timer @Pyjo.IOLoop.timer(3) def timeouter(loop): print('Timeout') # Shutdown server loop.remove(server) # Start event loop Pyjo.IOLoop.start()
Standalone HTTP server serving embedded template file¶
""" __DATA__ @@ index.html.ep <!DOCTYPE html> <html> <head> <meta charset="UTF-8"> <title>Pyjoyment</title> </head> <body> <h1>♥ Pyjoyment ♥</h1> <h2>This page is served by Pyjoyment framework.</h2> <p><%= method %> request for <%= path %></p> </body> </html> """ import Pyjo.App import Pyjo.GetOpt import Pyjo.Server.Daemon import Pyjo.Template import Pyjo.URL from Pyjo.Loader import data_section from Pyjo.Util import die class MyApp(Pyjo.App.object): # Embedded template file template = Pyjo.Template.new(auto_escape=True, vars=True).parse(data_section(__name__, 'index.html.ep')) # All the complexities of CGI, WSGI, ASGI, HTTP and WebSockets get reduced to a single method call! def handler(self, tx): # Resume transaction at the end of the block with tx: # Request method = tx.req.method path = tx.req.url.path # Response tx.res.code = 200 tx.res.headers.content_type = 'text/html; charset=utf-8' tx.res.body = self.template.process({'method': method, 'path': path}).encode() # Options --address and --port getopt = Pyjo.GetOpt.new().option('a|address=s', default='0.0.0.0').option('p|port=i', default=3000) opts = getopt.parse() or die('Usage: http_server.py [--address ADDRESS] [--port PORT]') url = Pyjo.URL.new().set(scheme='http', host=opts.address, port=opts.port) Pyjo.Server.Daemon.new(app=MyApp.new(), listen=[url]).run()
Requests on a timer¶
The user agent requests the server every second, in the same event loop, without blocking it:
import Pyjo.HelloWorld import Pyjo.IOLoop import Pyjo.Server.Daemon import Pyjo.UserAgent # The server and the user agent, in the same event loop, which close their sockets at the end of the block with ( Pyjo.Server.Daemon.new(app=Pyjo.HelloWorld.new(), listen=['http://*:3000']) as daemon, Pyjo.UserAgent.new() as ua, ): # A request every second, which does not block the event loop; a request that the end of the block interrupts gets # an error @Pyjo.IOLoop.recurring(1) def fetch(loop): def done(ua, tx): print(tx.error['message'] if tx.error else f'{tx.res.code} {tx.res.text}') ua.get('http://127.0.0.1:3000/', done) # The server runs the event loop until INT or TERM stops it daemon.run()
A request on a loopback¶
A blocking request of the user agent to the server, in the same event loop:
import Pyjo.HelloWorld import Pyjo.IOLoop import Pyjo.Server.Daemon import Pyjo.UserAgent # The server, on a port that the system picks, and the user agent, in the same event loop, which close their sockets at # the end of the block with ( Pyjo.Server.Daemon.new(app=Pyjo.HelloWorld.new(), listen=['http://127.0.0.1'], silent=True) as daemon, Pyjo.UserAgent.new(ioloop=Pyjo.IOLoop.singleton) as ua, ): # A blocking request, which runs the event loop of the server until the response tx = ua.get(f'http://127.0.0.1:{daemon.start().ports()[0]}/') print(tx.result.code, tx.result.text)
Composition¶
Quick start¶
A whole web application of Pyjoyment.Lite:
from Pyjoyment.Lite import * get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
JSON-RPC¶
A calculator with Bootstrap, which is a client of the server of JSON-RPC of the same application, whose methods validate their parameters with the schema of their type hints, and shows the request and the response of each call:
""" __DATA__ @@ index.html.ep <!DOCTYPE html> <html> <head> <title>JSON-RPC Calculator</title> %= stylesheet('/pyjo/bootstrap/bootstrap.css') </head> <body> <div class="container py-4"> <h1>JSON-RPC Calculator</h1> <form method="POST" action="<%= url_for('/') %>"> <div class="input-group mb-3"> <input class="form-control" type="number" step="any" name="a" value="<%= a %>" aria-label="First number" required> % for name, sign in operations.items(): <input class="btn-check" type="radio" name="method" id="<%= name %>" value="<%= name %>" autocomplete="off"<%= ' checked' if name == (method or 'add') else '' %>> <label class="btn btn-outline-primary" for="<%= name %>"><%= sign %></label> % end <input class="form-control" type="number" step="any" name="b" value="<%= b %>" aria-label="Second number" required> <button class="btn btn-primary" aria-label="Calculate">=</button> </div> </form> % if error: <div class="alert alert-danger" id="result"><%= error %></div> % elif method: <div class="alert alert-success" id="result"><%= a %> <%= operations[method] %> <%= b %> = <%= result %></div> % end % if request: <div class="row"> <div class="col-md-6 mb-3"> <h2 class="h6">Request</h2> <pre class="bg-body-tertiary border rounded p-2 text-wrap text-break" id="request"><%= request %></pre> </div> <div class="col-md-6 mb-3"> <h2 class="h6">Response</h2> <pre class="bg-body-tertiary border rounded p-2 text-wrap text-break" id="response"><%= response %></pre> </div> </div> % end </div> </body> </html> """ import Pyjo.Exception import Pyjo.JSON from Pyjo.JSON.RPC import RPC, Client, Error from Pyjoyment.Lite import * # The server of JSON-RPC, whose methods validate their parameters with the schema of their type hints rpc = RPC() # An error of the application, outside of the codes that JSON-RPC keeps for itself DIVISION_BY_ZERO = 1 OPERATIONS = {'add': '+', 'subtract': '−', 'multiply': '×', 'divide': '÷'} @rpc.method(schema=True) def add(a: float, b: float) -> float: return a + b @rpc.method(schema=True) def subtract(a: float, b: float) -> float: return a - b @rpc.method(schema=True) def multiply(a: float, b: float) -> float: return a * b @rpc.method(schema=True) def divide(a: float, b: float) -> float: if b == 0: raise Error.new(DIVISION_BY_ZERO, 'Division by zero') return a / b post('/rpc', rpc.to_action()) def number(value): # A number of the form, or the text itself, which the schema of the method rejects for convert in (int, float): try: return convert(value) except ValueError: pass return value def page(c, a='', b='', method=None, result=None, error=None, request=None, response=None): # The calculator, with the call and the exchange of JSON-RPC that gave its result method = method if method in OPERATIONS else None c.render( 'index', operations=OPERATIONS, a=a, b=b, method=method, result=result, error=error, request=request, response=response, ) get('/', page) # The calculator is a client of the server, through the user agent of the application, which requests the application # itself, as a relative URL has no host @post('/') async def calculate(c): a, b, method = c.param('a') or '', c.param('b') or '', c.param('method') exchange = {} def send(request): exchange['request'] = Pyjo.JSON.to_json(request) def received(tx): exchange['response'] = tx.res.text return tx.res.json() return c.ua().post_p('/rpc', json=request).then(received) result = error = None try: result = await Client.new(send).call_p(method or '', number(a), number(b)) # A number without a fraction, as 2.0 of a division, without it on the page, as in JavaScript if isinstance(result, float) and result.is_integer(): result = int(result) except Error as e: error = f'{e.message} ({e.code})' + (f': {e.data}' if e.data is not None else '') except Pyjo.Exception.object as e: error = e.message page(c, a, b, method, result, error, exchange.get('request'), exchange.get('response')) app.start()
Applications of other frameworks¶
A WSGI application, an ASGI application and an event function of AWS
Lambda, each in a file of its own, under the paths of an application of
Pyjoyment.Lite, on one server:
import Pyjo.App.Lambda from Pyjo.File import curfile from Pyjoyment.Lite import * # The WSGI application of hello_wsgi.py, the ASGI application of hello_asgi.py and the event function of AWS Lambda of # hello_lambda.py, next to this script, on one server, next to the routes of this application plugin('Mount', {'/wsgi': 'hello_wsgi:app'}) plugin('Mount', {'/asgi': 'hello_asgi:app'}) plugin('Mount', {'/lambda': Pyjo.App.Lambda.new(function='hello_lambda.handler', task_root=curfile().dirname())}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
A WSGI application, in a file of its own, under a path of an application of
Pyjoyment.Lite:
from Pyjoyment.Lite import * # The WSGI application of hello_wsgi.py, next to this script, under /wsgi plugin('Mount', {'/wsgi': 'hello_wsgi:app'}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
The WSGI application, which also runs on its own, on the WSGI server of the standard library:
from wsgiref.simple_server import make_server def app(environ, start_response): start_response('200 OK', [('Content-Type', 'text/plain')]) return [f'Hello WSGI at {environ["SCRIPT_NAME"]} for {environ["PATH_INFO"]}!'.encode()] if __name__ == '__main__': make_server('', 3000, app).serve_forever()
An application of Pyjoyment as the WSGI application, in the same file:
import Pyjo.Server.WSGI import Pyjoyment from Pyjoyment.Lite import * # Hello world of Pyjoyment as a WSGI application, whose script name is the path it is mounted under hello = Pyjoyment.new() hello.routes.get('/').to(cb=lambda c: c.render(text=f'Hello Pyjoyment at {c.url_for("/")} on WSGI!')) wsgi = Pyjo.Server.WSGI.new(app=hello).to_wsgi_app() # The WSGI application under /wsgi of the application of Pyjoyment.Lite plugin('Mount', {'/wsgi': wsgi}) get('/', {'text': 'I ♥ Pyjoyment!'}) # The application of a WSGI server too, which PYJO_WSGI tells: PYJO_WSGI=1 gunicorn mount_wsgi_inception:application application = app.start()
An ASGI application, in a file of its own, under a path of an application of
Pyjoyment.Lite, whose events stream at the pace of the client:
from Pyjoyment.Lite import * # The ASGI application of hello_asgi.py, next to this script, under /asgi plugin('Mount', {'/asgi': 'hello_asgi:app'}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
The ASGI application, which also runs on its own, on an ASGI server:
# On an ASGI server: uvicorn hello_asgi:app async def app(scope, receive, send): if scope['type'] != 'http': return # Server-sent events, each sent once the one before has been written if scope['path'].endswith('/events'): await send({'type': 'http.response.start', 'status': 200, 'headers': [(b'content-type', b'text/event-stream')]}) for n in range(3): await send({'type': 'http.response.body', 'body': f'data: {n}\n\n'.encode(), 'more_body': True}) await send({'type': 'http.response.body', 'body': b''}) return body = f'Hello ASGI at {scope["root_path"]} for {scope["path"]}!'.encode() await send({'type': 'http.response.start', 'status': 200, 'headers': [(b'content-type', b'text/plain')]}) await send({'type': 'http.response.body', 'body': body})
An application of Pyjoyment as the ASGI application, in the same file:
import Pyjo.Server.ASGI import Pyjoyment from Pyjoyment.Lite import * # Hello world of Pyjoyment as an ASGI application, whose root path is the path it is mounted under hello = Pyjoyment.new() hello.routes.get('/').to(cb=lambda c: c.render(text=f'Hello Pyjoyment at {c.url_for("/")} on ASGI!')) asgi = Pyjo.Server.ASGI.new(app=hello).to_asgi_app() # The ASGI application under /asgi of the application of Pyjoyment.Lite plugin('Mount', {'/asgi': asgi}) get('/', {'text': 'I ♥ Pyjoyment!'}) # The application of an ASGI server too, which PYJO_ASGI tells: PYJO_ASGI=1 uvicorn mount_asgi_inception:application application = app.start()
A server of the MCP SDK, whose transport of Streamable HTTP is an ASGI
application, under a path of an application of Pyjoyment.Lite:
from Pyjoyment.Lite import * from mcp.server.mcpserver import MCPServer # A server of the MCP SDK with a tool, whose transport of Streamable HTTP is an ASGI application of Starlette, under # /mcp/, as uv run --group servers python examples/mount/mount_mcp.py daemon serves it on http://localhost:3000/mcp/ mcp = MCPServer('Pyjoyment') @mcp.tool() def add(a: int, b: int) -> int: """Add two numbers.""" return a + b plugin('Mount', {'/mcp': mcp.streamable_http_app(streamable_http_path='/')}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
PHP on php-fpm, a FastCGI application, under a path of an application of
Pyjoyment.Lite, with the scripts of its directory and the pool of
php-fpm:
import Pyjo.App.FastCGI from Pyjo.File import curfile from Pyjoyment.Lite import * # The scripts of PHP of the directory php under /php, on php-fpm with the pool of php/php-fpm.conf next to this script: # php-fpm -n -y php/php-fpm.conf -F fastcgi = Pyjo.App.FastCGI.new(url='fcgi://127.0.0.1:9000', document_root=curfile().sibling('php').to_str()) plugin('Mount', {'/php': fastcgi}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()<?php // The script of the directory, and of every path under the mount when it is the only one, as a front controller is if ($_SERVER['PATH_INFO'] === '/events') { header('Content-Type: text/event-stream'); for ($n = 0; $n < 3; $n++) { echo "data: $n\n\n"; flush(); } exit; } header('Content-Type: text/plain'); echo "Hello PHP at {$_SERVER['SCRIPT_NAME']} for {$_SERVER['PATH_INFO']} ({$_SERVER['REQUEST_METHOD']} {$_SERVER['REQUEST_URI']})!\n";<?php phpinfo(); ?>; A pool of php-fpm in the foreground, for mount_fastcgi_php.py: php-fpm -n -y php/php-fpm.conf -F [global] error_log = /dev/stderr daemonize = no [www] listen = 127.0.0.1:9000 pm = static pm.max_children = 2
A CGI program of Python in the directory cgi-bin, under a path of an
application of Pyjoyment.Lite, a process for each request:
import Pyjo.App.CGI from Pyjo.File import curfile from Pyjoyment.Lite import * # The programs of the directory cgi-bin, next to this script, under /cgi-bin, a process for each request plugin('Mount', {'/cgi-bin': Pyjo.App.CGI.new(document_root=curfile().sibling('cgi-bin').to_str())}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()#!/usr/bin/env python3 # A CGI program, which writes its response: the headers, an empty line and the body import os import urllib.parse name = urllib.parse.parse_qs(os.environ.get('QUERY_STRING', '')).get('name', ['CGI'])[0] print('Content-Type: text/plain; charset=UTF-8') print() print(f'Hello {name} at {os.environ["SCRIPT_NAME"]} for {os.environ["PATH_INFO"]}!')
PHP on php-cgi, with the scripts of the same directory as for php-fpm, a process of php-cgi for each request:
import Pyjo.App.CGI from Pyjo.File import curfile from Pyjoyment.Lite import * # The scripts of PHP of the directory php under /php, on php-cgi of PATH, a process for each request cgi = Pyjo.App.CGI.new(command=['php-cgi'], document_root=curfile().sibling('php').to_str(), script_extensions=['.php']) plugin('Mount', {'/php': cgi}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
An event function of AWS Lambda, in a file of its own, under a path of an
application of Pyjoyment.Lite, in a process of its own, which gets the
requests one after another, as Lambda runs it:
import Pyjo.App.Lambda from Pyjo.File import curfile from Pyjoyment.Lite import * # The event function of hello_lambda.py, next to this script, under /lambda, in a process of its own, which gets the # requests one after another, as Lambda runs it plugin( 'Mount', {'/lambda': Pyjo.App.Lambda.new(function='hello_lambda.handler', task_root=curfile().dirname().to_str())} ) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
The event function, whose stage is the path it is mounted under:
# The handler "hello_lambda.handler" of an event function of AWS Lambda, of the payload format 2.0, whose stage is the # path it is mounted under def handler(event, context): stage, path = event['requestContext']['stage'], event['rawPath'] body = f'Hello Lambda at {stage} for {path}!' return {'statusCode': 200, 'headers': {'content-type': 'text/plain'}, 'body': body}
Starlette on Mangum as the event function, which runs an event loop of its own in its process:
import Pyjo.App.Lambda from Pyjo.File import curfile from Pyjoyment.Lite import * # Starlette on Mangum, the event function of hello_mangum.py, next to this script, under /mangum, which runs an event # loop of its own in its process plugin( 'Mount', {'/mangum': Pyjo.App.Lambda.new(function='hello_mangum.handler', task_root=curfile().dirname().to_str())} ) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()from mangum import Mangum from starlette.applications import Starlette from starlette.responses import PlainTextResponse from starlette.routing import Route async def hello(request): return PlainTextResponse(f'Hello Starlette for {request.url.path} on Mangum!') app = Starlette(routes=[Route('/{path:path}', hello)]) # The handler "hello_mangum.handler" of an event function of AWS Lambda, with the path that it is mounted under, the # stage, as the base path, which Mangum leaves out handler = Mangum(app, lifespan='off', api_gateway_base_path='/mangum')
An application of Pyjoyment on Pyjo.Server.Lambda as the event function,
in the same file, which its process imports:
import Pyjo.App.Lambda import Pyjo.Server.Lambda import Pyjoyment from Pyjo.File import curfile from Pyjoyment.Lite import * # Hello world of Pyjoyment as an event function of AWS Lambda, whose stage is the path it is mounted under, as its base # path hello = Pyjoyment.new() hello.routes.get('/').to(cb=lambda c: c.render(text=f'Hello Pyjoyment at {c.url_for("/")} on Lambda!')) handler = Pyjo.Server.Lambda.new(app=hello).to_lambda_app() # The handler above under /lambda of the application of Pyjoyment.Lite, in a process of its own, which imports this # script as the module of the function function = Pyjo.App.Lambda.new(function='mount_lambda_inception.handler', task_root=curfile().dirname().to_str()) plugin('Mount', {'/lambda': function}) get('/', {'text': 'I ♥ Pyjoyment!'}) app.start()
Both sides of the interface¶
Servers¶
Pyjo.HelloWorld natively, on the web server of Pyjoyment:
import Pyjo.HelloWorld import Pyjo.Server.Daemon Pyjo.Server.Daemon.new(app=Pyjo.HelloWorld.new(), listen=['http://*:3000']).run()
On uvicorn, an ASGI server:
import Pyjo.HelloWorld import Pyjo.Server.ASGI import uvicorn app = Pyjo.Server.ASGI.new(app=Pyjo.HelloWorld.new()).to_asgi_app() uvicorn.run(app, port=3000)
On Werkzeug, a WSGI server:
import Pyjo.HelloWorld import Pyjo.Server.WSGI from werkzeug.serving import run_simple app = Pyjo.Server.WSGI.new(app=Pyjo.HelloWorld.new()).to_wsgi_app() run_simple('localhost', 3000, app)
A project of Django in one file, as a
WSGI application on the server of Pyjoyment, whose views request the API of
GitHub with Pyjo.UserAgent, as Django describes it:
import Pyjo.UserAgent from Pyjoyment.Lite import * import os from django.conf import settings from django.core.wsgi import get_wsgi_application from django.http import HttpResponse, JsonResponse from django.urls import path # A project of Django in one file settings.configure(ROOT_URLCONF=__name__, ALLOWED_HOSTS=['*'], SECRET_KEY='insecure', TIME_ZONE='UTC') # The user agent of the views, in the place of requests, and the API of GitHub, or the one of GITHUB_API_URL ua = Pyjo.UserAgent.new() api = os.environ.get('GITHUB_API_URL', 'https://api.github.com') def index(request): return HttpResponse(f'Hello from Django at {request.path}!') # A blocking request, which runs an event loop of its own, while the worker waits for it as for a query of the ORM, as # the application sees no running event loop def release(request): res = ua.get(f'{api}/repos/pyjoyment/Pyjoyment/releases/latest').result return JsonResponse({'status': res.code, 'release': res.json('/tag_name')}) urlpatterns = [path('', index), path('release', release)] plugin('Mount', {'/': get_wsgi_application()}) app.start()
The same as an ASGI application, whose views await the requests:
import Pyjo.UserAgent from Pyjoyment.Lite import * import os from django.conf import settings from django.core.asgi import get_asgi_application from django.http import HttpResponse, JsonResponse from django.urls import path # A project of Django in one file settings.configure(ROOT_URLCONF=__name__, ALLOWED_HOSTS=['*'], SECRET_KEY='insecure', TIME_ZONE='UTC') # The user agent of the views, in the place of httpx, and the API of GitHub, or the one of GITHUB_API_URL ua = Pyjo.UserAgent.new() api = os.environ.get('GITHUB_API_URL', 'https://api.github.com') def index(request): return HttpResponse(f'Hello from Django at {request.path}!') # A request awaited in the event loop of asyncio, which serves the other requests meanwhile async def release(request): res = (await ua.get_p(f'{api}/repos/pyjoyment/Pyjoyment/releases/latest')).result return JsonResponse({'status': res.code, 'release': res.json('/tag_name')}) urlpatterns = [path('', index), path('release', release)] # An ASGI application, on the reactor of asyncio, the default one plugin('Mount', {'/': get_asgi_application()}) app.start()
Deployment¶
AWS Lambda¶
As an event function of AWS Lambda:
import Pyjo.HelloWorld import Pyjo.Server.Lambda # The handler "hello_world_lambda.handler" of an event function of AWS Lambda handler = Pyjo.Server.Lambda.new(app=Pyjo.HelloWorld.new()).to_lambda_app()
On a custom runtime of AWS Lambda, as provided.al2023, its bootstrap
runs it with the runtime client of Pyjoyment:
#!/bin/sh # The bootstrap of a custom runtime of AWS Lambda, as provided.al2023, which runs the handler of the function, as # "hello_world_lambda.handler" of _HANDLER, with the runtime client of Pyjoyment exec python3 -m Pyjo.Server.Lambda
Integration¶
WebSocket¶
A page of the application opens a WebSocket to it and sends it a URL, which the application requests with its user agent, without blocking the other connections, and sends back the title of the page:
""" __DATA__ @@ index.html.ep % url = url_for('title') <script> const ws = new WebSocket('<%= url.to_abs() %>'); ws.onmessage = function (event) { document.body.innerHTML += event.data }; ws.onopen = function (event) { ws.send('https://mojolicious.org') }; </script> """ from Pyjoyment.Lite import * # Render template "index.html.ep" from the DATA section @get('/') def index(c): c.render(template='index') # WebSocket service used by the template to extract the title from a website @websocket('/title') def title(c): @c.on('message') def message(c, msg): c.ua().get_p(msg).then(lambda tx: c.send(tx.result.dom().at('title').text)) app.start()
Amazon S3¶
A bucket of Amazon S3 as a website, with aiobotocore, the asynchronous AWS SDK, whose
requests the actions await in the event loop of the server: the content of an
object, as it comes, index.html of a folder, or the list of its folders
and objects:
""" __DATA__ @@ folder.html.ep <!DOCTYPE html> <html> <head><title>/<%= prefix %></title></head> <body> <h1>/<%= prefix %></h1> <ul> % if prefix: <li><a href="../">../</a></li> % end % for name in names: <li><a href="<%= url_for('object', key=prefix + name) %>"><%= name %></a></li> % end </ul> </body> </html> """ import Pyjo.Date import Pyjo.GetOpt import Pyjo.Promise from Pyjo.Util import die, getopt from Pyjoyment.Lite import * import contextlib import os import threading # The asynchronous AWS SDK, on the event loop of asyncio, which the default reactor runs import aiobotocore.session import botocore.exceptions # The options before the command, with the names of the AWS CLI, which set the environment variables that the # application and the AWS SDK read: python s3.py --profile myprofile --bucket mybucket daemon getopt( { 'bucket=s': Pyjo.GetOpt.env('S3_BUCKET'), 'endpoint-url=s': Pyjo.GetOpt.env('AWS_ENDPOINT_URL'), 'profile=s': Pyjo.GetOpt.env('AWS_PROFILE'), 'region=s': Pyjo.GetOpt.env('AWS_REGION'), }, pass_through=True, require_order=True, ) # The bucket, whose objects the application serves, with the credentials and the region of the AWS SDK bucket = os.environ.get('S3_BUCKET') or die( 'Usage: s3.py --bucket BUCKET [--profile PROFILE] [--region REGION] [--endpoint-url URL] COMMAND [OPTIONS]' ) session = aiobotocore.session.get_session() # A client of S3 for each event loop, as its connections belong to one, which the first request that needs it makes local = threading.local() def s3_p(): if not hasattr(local, 'client_p'): local.clients = contextlib.AsyncExitStack() local.client_p = Pyjo.Promise.new(local.clients.enter_async_context(session.create_client('s3'))) return local.client_p # The client closes its connections once the server stops @hook('after_server_stop') def after_server_stop(server, app): if hasattr(local, 'clients'): del local.client_p return Pyjo.Promise.new(local.clients.aclose()) def not_found(error): return error.response['Error']['Code'] in ('NoSuchKey', '404') # A promise of a write, which the client has read once it is fulfilled def write_p(c, chunk): drained = Pyjo.Promise.new() c.write(chunk, lambda c: drained.resolve(True)) return drained # The content of an object, as it comes, each chunk once the client has read the one before async def send_object(c, key): s3 = await s3_p() response = await s3.get_object(Bucket=bucket, Key=key) headers = c.res.headers headers.content_type = response['ContentType'] headers.content_length = response['ContentLength'] headers.etag = response['ETag'] headers.last_modified = Pyjo.Date.new(response['LastModified']).to_str() # The response ends with its last byte, as it has the length of the object, and a client that goes away finishes the # transaction, which drains no more finished = Pyjo.Promise.new() c.on('finish', lambda c: finished.resolve(False)) async with response['Body'] as body: if not response['ContentLength']: return c.rendered() async for chunk in body.iter_chunks(64 * 1024): if not await Pyjo.Promise.race(write_p(c, chunk), finished): return # The folders and the objects under a prefix, as S3 has no folders, only keys with slashes, and a folder that the console # of S3 makes, an empty object of the prefix, which the list leaves out async def send_folder(c, prefix): s3 = await s3_p() names, count = [], 0 async for page in s3.get_paginator('list_objects_v2').paginate(Bucket=bucket, Prefix=prefix, Delimiter='/'): count += page['KeyCount'] names += [p['Prefix'].removeprefix(prefix) for p in page.get('CommonPrefixes', [])] names += [o['Key'].removeprefix(prefix) for o in page.get('Contents', []) if o['Key'] != prefix] if prefix and not count: return c.reply.not_found() c.render('folder', prefix=prefix, names=names) # An object, "index.html" or the list of a folder, for a key that ends with a slash @get('/*key', {'key': ''}, 'object') async def serve(c): key = c.param('key') if key and not key.endswith('/'): try: return await send_object(c, key) except botocore.exceptions.ClientError as e: if not not_found(e): raise # A folder without its slash s3 = await s3_p() if (await s3.list_objects_v2(Bucket=bucket, Prefix=f'{key}/', MaxKeys=1))['KeyCount']: return c.redirect_to('object', key=f'{key}/') return c.reply.not_found() try: await send_object(c, f'{key}index.html') except botocore.exceptions.ClientError as e: if not not_found(e): raise await send_folder(c, key) app.start()