diff --git a/docker/config/nginx/sapl.conf b/docker/config/nginx/sapl.conf index f7e0603e2..0102d6476 100644 --- a/docker/config/nginx/sapl.conf +++ b/docker/config/nginx/sapl.conf @@ -97,20 +97,18 @@ server { limit_req zone=sapl_general burst=${NGINX_BURST_API} nodelay; limit_req_status 429; - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, HEAD, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'Access-Control-Allow-Origin,XMLHttpRequest,Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Mx-ReqToken,X-Requested-With'; - add_header 'Access-Control-Expose-Headers' 'Access-Control-Allow-Origin,XMLHttpRequest,Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Mx-ReqToken,X-Requested-With'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, HEAD, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization,Accept,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Expose-Headers' 'Content-Type,X-RateLimit-Reason,Retry-After,X-Quota-Daily-Remaining,X-Quota-Weekly-Remaining' always; if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, HEAD, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'Authorization,Accept,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; - add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain; charset=utf-8'; - add_header 'Content-Length' 0; + add_header 'Access-Control-Allow-Headers' 'Authorization,Accept,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; return 204; } diff --git a/plan/RATE-LIMITER-PLAN.md b/plan/RATE-LIMITER-PLAN.md index ddaac7229..458df0014 100644 --- a/plan/RATE-LIMITER-PLAN.md +++ b/plan/RATE-LIMITER-PLAN.md @@ -110,6 +110,10 @@ graph TD | Path counter | `rl:{ns}:path:{sha256}:reqs` | 60 s | 1 | ~0.3 MB | | UA deny list | `rl:bot:ua:blocked` | permanent SET | 1 | ~0.03 MB | | NS/IP/window counter | `rl:{ns}:ip:{ip}:w:{bucket}` | 120 s | 1 | ~0.6 MB | +| API daily quota (anon) | `quota:{ns}:daily:{date}:ip:{ip}` | 24 h | 1 | negligible | +| API weekly quota (anon) | `quota:{ns}:weekly:{week}:ip:{ip}` | 7 d | 1 | negligible | +| API daily quota (auth) | `quota:{ns}:daily:{date}:user:{uid}` | 24 h | 1 | negligible | +| API weekly quota (auth) | `quota:{ns}:weekly:{week}:user:{uid}` | 7 d | 1 | negligible | | Redis overhead (× 1.5) | | | | ~1.6 GB | | **Total ceiling** | | | | **~5 GB** | @@ -615,9 +619,12 @@ When the window count hits the threshold the IP/user is written to a Redis blocked-set with a 300 s TTL and subsequent requests return 429 with `Retry-After: 300` — without touching the database. -Decision flow inside `RateLimitMiddleware._evaluate()`: +Decision flow inside `RateLimitMiddleware.__call__()` / `_evaluate()`: ``` +0. /api/ + path AND consumer daily/weekly quota exceeded? → 429 reason=quota_daily / quota_weekly + (per-consumer: auth users by pk, anon by masked IP; fail-open when Redis unavailable) 1. IP in whitelist? → pass (no further checks) 1a. UA matches BOT_UA_FRAGMENTS list? → 429 reason=known_ua 1b. UA token hash in rl:bot:ua:blocked SET? → 429 reason=redis_ua @@ -639,7 +646,11 @@ Decision flow inside `RateLimitMiddleware._evaluate()`: ```mermaid flowchart TD - REQ([Request]) --> C1 + REQ([Request]) --> C0 + + C0{"/api/ path AND\ndaily/weekly quota exceeded?"} + C0 -- "yes — quota:{ns}:daily/weekly:{period}:user/ip exceeded" --> R_QUOTA([429\nquota_daily / quota_weekly]) + C0 -- no --> C1 C1{"Known bot UA?"} C1 -- "yes — substring in BOT_UA_FRAGMENTS" --> R_UA([429\nknown_ua]) @@ -696,6 +707,7 @@ Roll out to canary pods first; promote check-by-check in order of false-positive | Order | Check | Reason | Risk | Condition to promote | |-------|-------|--------|------|---------------------| | nginx | scanner extensions | `return 444` in `sapl.conf` for `.php`/`.asp`/etc. | Zero | Gunicorn never sees these requests | +| 0th | `quota_daily` / `quota_weekly` | Per-consumer daily/weekly cap on `/api/` paths | Low | Limits set well above per-minute rate (200/day anon, 1000/day auth) | | 1st | `known_ua` | Substring in hardcoded `BOT_UA_FRAGMENTS` list | Zero | UA strings are deterministic | | 2nd | `redis_ua` | Token hash in `rl:bot:ua:blocked` SET | Zero | Keys only set manually by operators | | 3rd | `ip_blocked` | Marker set by prior proven-bad requests | Zero | Fast-path only, no new blocks created | @@ -866,6 +878,10 @@ Redis PDF caching would solve "high request volume reaching the file layer" — | 1 | Path counter (`/media/`) | `rl:{ns}:path:{sha256}:reqs` | 60 s | — (observability only) | `RL_PATH_REQUESTS` | | 1 | Path counter (`/static/`) | `rl:{ns}:path:{sha256}:reqs` | 60 s | — | *Future* (requires OpenResty/Lua) | | 1 | UA deny list | `rl:bot:ua:blocked` | permanent SET | — (block on match) | `RL_UA_BLOCKLIST` | +| 1 | API daily quota (anon) | `quota:{ns}:daily:{date}:ip:{ip}` | 24 h | 50 (`API_QUOTA_ANON_DAILY`) | `QUOTA_IP_DAILY` | +| 1 | API weekly quota (anon) | `quota:{ns}:weekly:{week}:ip:{ip}` | 7 d | 350 (`API_QUOTA_ANON_WEEKLY`) | `QUOTA_IP_WEEKLY` | +| 1 | API daily quota (auth) | `quota:{ns}:daily:{date}:user:{uid}` | 24 h | 1000 (`API_QUOTA_AUTH_DAILY`) | `QUOTA_USER_DAILY` | +| 1 | API weekly quota (auth) | `quota:{ns}:weekly:{week}:user:{uid}` | 7 d | 7000 (`API_QUOTA_AUTH_WEEKLY`) | `QUOTA_USER_WEEKLY` | | 2 | Django Channels | `channels:*` | session TTL | — | *Future* | ### What each counter catches — and misses diff --git a/sapl/middleware/ratelimit.py b/sapl/middleware/ratelimit.py index 65efaace6..bb93ad6e0 100644 --- a/sapl/middleware/ratelimit.py +++ b/sapl/middleware/ratelimit.py @@ -2,6 +2,7 @@ RateLimitMiddleware — cross-pod rate limiting backed by shared Redis. Decision flow (per request): + 0. /api/ path AND consumer daily/weekly quota exceeded? → 429 1. Known bot UA? → 429 (Python list — substring match) 1b. Redis UA deny list? → 429 (runtime SET — token hash match, refreshed every 60 s) 2. Anonymous AND IP in blocked set? → 429 (authenticated users skip — have per-user limit at 3c) @@ -59,6 +60,18 @@ RL_PATH_REQUESTS = 'rl:{ns}:path:{sha256}:reqs' RL_UA_BLOCKLIST = 'rl:bot:ua:blocked' # permanent SET — runtime UA deny list RL_METRICS_BLOCKED = 'rl:metrics:{ns}:{date}:blocked:{reason}' # daily counter per block reason +# --------------------------------------------------------------------------- +# API quota keys — per-consumer, per-day/week, tenant-scoped. +# Consumer identity: authenticated users by uid, anonymous by masked IP. +# Weekly key uses ISO week notation (yyyy-Www) — unambiguous, Monday-anchored. +# TTL set only on first INCR (Lua); daily=24h, weekly=7d — cleanup only, +# resets are implicit in the date/week embedded in the key name. +# --------------------------------------------------------------------------- +QUOTA_USER_DAILY = 'quota:{ns}:daily:{date}:user:{uid}' +QUOTA_USER_WEEKLY = 'quota:{ns}:weekly:{week}:user:{uid}' +QUOTA_IP_DAILY = 'quota:{ns}:daily:{date}:ip:{ip}' +QUOTA_IP_WEEKLY = 'quota:{ns}:weekly:{week}:ip:{ip}' + # --------------------------------------------------------------------------- # Bot UA fragments # --------------------------------------------------------------------------- @@ -215,6 +228,10 @@ class RateLimitMiddleware: self._bypass_paths = [ re.compile(p) for p in getattr(settings, 'RATE_LIMIT_BYPASS_PATHS', []) ] + self.api_quota_anon_daily = settings.API_QUOTA_ANON_DAILY + self.api_quota_anon_weekly = settings.API_QUOTA_ANON_WEEKLY + self.api_quota_auth_daily = settings.API_QUOTA_AUTH_DAILY + self.api_quota_auth_weekly = settings.API_QUOTA_AUTH_WEEKLY logger.info( '[RATELIMIT] anon=%s auth=%s bot=%s whitelist=%s bypass_paths=%s', settings.RATE_LIMITER_RATE, @@ -223,11 +240,33 @@ class RateLimitMiddleware: list(self.whitelist) or '(none)', [p.pattern for p in self._bypass_paths] or '(none)', ) + logger.info( + '[API QUOTAS] daily_anon=%s weekly_anon=%s daily_auth=%s weekly_auth=%s', + settings.API_QUOTA_ANON_DAILY, + settings.API_QUOTA_ANON_WEEKLY, + settings.API_QUOTA_AUTH_DAILY, + settings.API_QUOTA_AUTH_WEEKLY, + ) def __call__(self, request): if any(p.match(request.path) for p in self._bypass_paths): return self.get_response(request) + if request.path.startswith('/api/'): + exceeded = self._check_api_quota(request) + if exceeded: + ip = get_client_ip(request) + logger.warning( + 'quota_exceeded window=%s ip=%s path=%s namespace=%s', + exceeded, ip, request.path, _NAMESPACE, + extra={'ua': request.META.get('HTTP_USER_AGENT', '')}, + ) + self._inc_block_metric(f'quota_{exceeded}') + response = HttpResponse(status=429) + response['Retry-After'] = 86400 + response['X-RateLimit-Reason'] = f'quota_{exceeded}' + return response + decision = self._evaluate(request) if decision['action'] == 'block': logger.warning( @@ -241,6 +280,7 @@ class RateLimitMiddleware: self._inc_block_metric(decision['reason']) response = HttpResponse(status=429) response['Retry-After'] = self.BLOCK_TTL + response['X-RateLimit-Reason'] = decision['reason'] return response logger.debug( 'ratelimit_pass ip=%s path=%s user=%s namespace=%s', @@ -357,6 +397,41 @@ class RateLimitMiddleware: ) self._inc_block_metric('404_scan') + def _check_api_quota(self, request): + """ + Increment per-consumer daily and weekly API quota counters. + Returns 'daily' or 'weekly' if the respective limit is exceeded, else None. + Fails open (returns None) if Redis/cache is unavailable. + Consumer identity: authenticated users by pk, anonymous by masked IP. + """ + today = date.today() + iso = today.isocalendar() + date_str = today.isoformat() + week_str = f'{iso[0]}-W{iso[1]:02d}' + + user = getattr(request, 'user', None) + if user and user.is_authenticated: + uid = str(user.pk) + d_key = QUOTA_USER_DAILY.format(ns=_NAMESPACE, date=date_str, uid=uid) + w_key = QUOTA_USER_WEEKLY.format(ns=_NAMESPACE, week=week_str, uid=uid) + d_limit = self.api_quota_auth_daily + w_limit = self.api_quota_auth_weekly + else: + ip = get_client_ip(request) + d_key = QUOTA_IP_DAILY.format(ns=_NAMESPACE, date=date_str, ip=ip) + w_key = QUOTA_IP_WEEKLY.format(ns=_NAMESPACE, week=week_str, ip=ip) + d_limit = self.api_quota_anon_daily + w_limit = self.api_quota_anon_weekly + + try: + if _incr_with_ttl(d_key, 86400) > d_limit: + return 'daily' + if _incr_with_ttl(w_key, 7 * 86400) > w_limit: + return 'weekly' + except Exception: + pass # fail open — quota not enforced when Redis unavailable + return None + def _incr_with_ttl(self, key, ttl): return _incr_with_ttl(key, ttl) diff --git a/sapl/settings.py b/sapl/settings.py index 1b2f53875..781d86cbf 100644 --- a/sapl/settings.py +++ b/sapl/settings.py @@ -433,6 +433,17 @@ RATE_LIMIT_BYPASS_PATHS = [ r'^/painel/\d+/dados$', ] +# API quota — daily and weekly call caps per consumer (Redis-only, no DB migration). +# Applied only to /api/ paths. Per-consumer: auth users by pk, anon by IP. +# Weekly default = 7 × daily (daily limit is the binding constraint). +# Anon quota is tighter than auth quota — mirrors the rate limiter relationship. +# Both must be > their respective per-minute rate limit thresholds (35 anon, 120 auth), +# otherwise the quota fires before the rate limiter ever engages. +API_QUOTA_ANON_DAILY = config('API_QUOTA_ANON_DAILY', default=50, cast=int) +API_QUOTA_ANON_WEEKLY = config('API_QUOTA_ANON_WEEKLY', default=350, cast=int) +API_QUOTA_AUTH_DAILY = config('API_QUOTA_AUTH_DAILY', default=1000, cast=int) +API_QUOTA_AUTH_WEEKLY = config('API_QUOTA_AUTH_WEEKLY', default=7000, cast=int) + # Media file serving — serve_media (sapl/base/media.py) via X-Accel-Redirect. # TTL for both URL-path and storage-path access counters (DB 1). MEDIA_PATH_COUNTER_TTL = config('MEDIA_PATH_COUNTER_TTL', default=60, cast=int)