|
|
@ -2,6 +2,7 @@ |
|
|
RateLimitMiddleware — cross-pod rate limiting backed by shared Redis. |
|
|
RateLimitMiddleware — cross-pod rate limiting backed by shared Redis. |
|
|
|
|
|
|
|
|
Decision flow (per request): |
|
|
Decision flow (per request): |
|
|
|
|
|
0. /api/ path AND consumer daily/weekly quota exceeded? → 429 |
|
|
1. Known bot UA? → 429 (Python list — substring match) |
|
|
1. Known bot UA? → 429 (Python list — substring match) |
|
|
1b. Redis UA deny list? → 429 (runtime SET — token hash match, refreshed every 60 s) |
|
|
1b. Redis UA deny list? → 429 (runtime SET — token hash match, refreshed every 60 s) |
|
|
2. Anonymous AND IP in blocked set? → 429 (authenticated users skip — have per-user limit at 3c) |
|
|
2. Anonymous AND IP in blocked set? → 429 (authenticated users skip — have per-user limit at 3c) |
|
|
@ -59,6 +60,18 @@ RL_PATH_REQUESTS = 'rl:{ns}:path:{sha256}:reqs' |
|
|
RL_UA_BLOCKLIST = 'rl:bot:ua:blocked' # permanent SET — runtime UA deny list |
|
|
RL_UA_BLOCKLIST = 'rl:bot:ua:blocked' # permanent SET — runtime UA deny list |
|
|
RL_METRICS_BLOCKED = 'rl:metrics:{ns}:{date}:blocked:{reason}' # daily counter per block reason |
|
|
RL_METRICS_BLOCKED = 'rl:metrics:{ns}:{date}:blocked:{reason}' # daily counter per block reason |
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
|
|
|
|
# API quota keys — per-consumer, per-day/week, tenant-scoped. |
|
|
|
|
|
# Consumer identity: authenticated users by uid, anonymous by masked IP. |
|
|
|
|
|
# Weekly key uses ISO week notation (yyyy-Www) — unambiguous, Monday-anchored. |
|
|
|
|
|
# TTL set only on first INCR (Lua); daily=24h, weekly=7d — cleanup only, |
|
|
|
|
|
# resets are implicit in the date/week embedded in the key name. |
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
|
|
|
|
QUOTA_USER_DAILY = 'quota:{ns}:daily:{date}:user:{uid}' |
|
|
|
|
|
QUOTA_USER_WEEKLY = 'quota:{ns}:weekly:{week}:user:{uid}' |
|
|
|
|
|
QUOTA_IP_DAILY = 'quota:{ns}:daily:{date}:ip:{ip}' |
|
|
|
|
|
QUOTA_IP_WEEKLY = 'quota:{ns}:weekly:{week}:ip:{ip}' |
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
|
# --------------------------------------------------------------------------- |
|
|
# Bot UA fragments |
|
|
# Bot UA fragments |
|
|
# --------------------------------------------------------------------------- |
|
|
# --------------------------------------------------------------------------- |
|
|
@ -215,6 +228,10 @@ class RateLimitMiddleware: |
|
|
self._bypass_paths = [ |
|
|
self._bypass_paths = [ |
|
|
re.compile(p) for p in getattr(settings, 'RATE_LIMIT_BYPASS_PATHS', []) |
|
|
re.compile(p) for p in getattr(settings, 'RATE_LIMIT_BYPASS_PATHS', []) |
|
|
] |
|
|
] |
|
|
|
|
|
self.api_quota_anon_daily = settings.API_QUOTA_ANON_DAILY |
|
|
|
|
|
self.api_quota_anon_weekly = settings.API_QUOTA_ANON_WEEKLY |
|
|
|
|
|
self.api_quota_auth_daily = settings.API_QUOTA_AUTH_DAILY |
|
|
|
|
|
self.api_quota_auth_weekly = settings.API_QUOTA_AUTH_WEEKLY |
|
|
logger.info( |
|
|
logger.info( |
|
|
'[RATELIMIT] anon=%s auth=%s bot=%s whitelist=%s bypass_paths=%s', |
|
|
'[RATELIMIT] anon=%s auth=%s bot=%s whitelist=%s bypass_paths=%s', |
|
|
settings.RATE_LIMITER_RATE, |
|
|
settings.RATE_LIMITER_RATE, |
|
|
@ -223,11 +240,33 @@ class RateLimitMiddleware: |
|
|
list(self.whitelist) or '(none)', |
|
|
list(self.whitelist) or '(none)', |
|
|
[p.pattern for p in self._bypass_paths] or '(none)', |
|
|
[p.pattern for p in self._bypass_paths] or '(none)', |
|
|
) |
|
|
) |
|
|
|
|
|
logger.info( |
|
|
|
|
|
'[API QUOTAS] daily_anon=%s weekly_anon=%s daily_auth=%s weekly_auth=%s', |
|
|
|
|
|
settings.API_QUOTA_ANON_DAILY, |
|
|
|
|
|
settings.API_QUOTA_ANON_WEEKLY, |
|
|
|
|
|
settings.API_QUOTA_AUTH_DAILY, |
|
|
|
|
|
settings.API_QUOTA_AUTH_WEEKLY, |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
def __call__(self, request): |
|
|
def __call__(self, request): |
|
|
if any(p.match(request.path) for p in self._bypass_paths): |
|
|
if any(p.match(request.path) for p in self._bypass_paths): |
|
|
return self.get_response(request) |
|
|
return self.get_response(request) |
|
|
|
|
|
|
|
|
|
|
|
if request.path.startswith('/api/'): |
|
|
|
|
|
exceeded = self._check_api_quota(request) |
|
|
|
|
|
if exceeded: |
|
|
|
|
|
ip = get_client_ip(request) |
|
|
|
|
|
logger.warning( |
|
|
|
|
|
'quota_exceeded window=%s ip=%s path=%s namespace=%s', |
|
|
|
|
|
exceeded, ip, request.path, _NAMESPACE, |
|
|
|
|
|
extra={'ua': request.META.get('HTTP_USER_AGENT', '')}, |
|
|
|
|
|
) |
|
|
|
|
|
self._inc_block_metric(f'quota_{exceeded}') |
|
|
|
|
|
response = HttpResponse(status=429) |
|
|
|
|
|
response['Retry-After'] = 86400 |
|
|
|
|
|
response['X-RateLimit-Reason'] = f'quota_{exceeded}' |
|
|
|
|
|
return response |
|
|
|
|
|
|
|
|
decision = self._evaluate(request) |
|
|
decision = self._evaluate(request) |
|
|
if decision['action'] == 'block': |
|
|
if decision['action'] == 'block': |
|
|
logger.warning( |
|
|
logger.warning( |
|
|
@ -241,6 +280,7 @@ class RateLimitMiddleware: |
|
|
self._inc_block_metric(decision['reason']) |
|
|
self._inc_block_metric(decision['reason']) |
|
|
response = HttpResponse(status=429) |
|
|
response = HttpResponse(status=429) |
|
|
response['Retry-After'] = self.BLOCK_TTL |
|
|
response['Retry-After'] = self.BLOCK_TTL |
|
|
|
|
|
response['X-RateLimit-Reason'] = decision['reason'] |
|
|
return response |
|
|
return response |
|
|
logger.debug( |
|
|
logger.debug( |
|
|
'ratelimit_pass ip=%s path=%s user=%s namespace=%s', |
|
|
'ratelimit_pass ip=%s path=%s user=%s namespace=%s', |
|
|
@ -357,6 +397,41 @@ class RateLimitMiddleware: |
|
|
) |
|
|
) |
|
|
self._inc_block_metric('404_scan') |
|
|
self._inc_block_metric('404_scan') |
|
|
|
|
|
|
|
|
|
|
|
def _check_api_quota(self, request): |
|
|
|
|
|
""" |
|
|
|
|
|
Increment per-consumer daily and weekly API quota counters. |
|
|
|
|
|
Returns 'daily' or 'weekly' if the respective limit is exceeded, else None. |
|
|
|
|
|
Fails open (returns None) if Redis/cache is unavailable. |
|
|
|
|
|
Consumer identity: authenticated users by pk, anonymous by masked IP. |
|
|
|
|
|
""" |
|
|
|
|
|
today = date.today() |
|
|
|
|
|
iso = today.isocalendar() |
|
|
|
|
|
date_str = today.isoformat() |
|
|
|
|
|
week_str = f'{iso[0]}-W{iso[1]:02d}' |
|
|
|
|
|
|
|
|
|
|
|
user = getattr(request, 'user', None) |
|
|
|
|
|
if user and user.is_authenticated: |
|
|
|
|
|
uid = str(user.pk) |
|
|
|
|
|
d_key = QUOTA_USER_DAILY.format(ns=_NAMESPACE, date=date_str, uid=uid) |
|
|
|
|
|
w_key = QUOTA_USER_WEEKLY.format(ns=_NAMESPACE, week=week_str, uid=uid) |
|
|
|
|
|
d_limit = self.api_quota_auth_daily |
|
|
|
|
|
w_limit = self.api_quota_auth_weekly |
|
|
|
|
|
else: |
|
|
|
|
|
ip = get_client_ip(request) |
|
|
|
|
|
d_key = QUOTA_IP_DAILY.format(ns=_NAMESPACE, date=date_str, ip=ip) |
|
|
|
|
|
w_key = QUOTA_IP_WEEKLY.format(ns=_NAMESPACE, week=week_str, ip=ip) |
|
|
|
|
|
d_limit = self.api_quota_anon_daily |
|
|
|
|
|
w_limit = self.api_quota_anon_weekly |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
if _incr_with_ttl(d_key, 86400) > d_limit: |
|
|
|
|
|
return 'daily' |
|
|
|
|
|
if _incr_with_ttl(w_key, 7 * 86400) > w_limit: |
|
|
|
|
|
return 'weekly' |
|
|
|
|
|
except Exception: |
|
|
|
|
|
pass # fail open — quota not enforced when Redis unavailable |
|
|
|
|
|
return None |
|
|
|
|
|
|
|
|
def _incr_with_ttl(self, key, ttl): |
|
|
def _incr_with_ttl(self, key, ttl): |
|
|
return _incr_with_ttl(key, ttl) |
|
|
return _incr_with_ttl(key, ttl) |
|
|
|
|
|
|
|
|
|
