|
|
@ -81,14 +81,16 @@ RL_API_IP_BLOCKED = 'rl:api:ns:{ns}:ip:{ip}:blocked' |
|
|
RL_INDEX_API_BLOCKED_IPS = 'rl:index:api_blocked_ips' |
|
|
RL_INDEX_API_BLOCKED_IPS = 'rl:index:api_blocked_ips' |
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
|
# --------------------------------------------------------------------------- |
|
|
# API quota keys — per-consumer, per-day/week, tenant-scoped. |
|
|
# API quota keys — per-tenant HASH, one key per period. |
|
|
# Consumer identity: authenticated users by uid, anonymous by masked IP. |
|
|
# Structure: HASH key = quota:{ns}:daily:{date} field = {ip} value = counter |
|
|
|
|
|
# HASH key = quota:{ns}:weekly:{week} field = {ip} value = counter |
|
|
# Weekly key uses ISO week notation (yyyy-Www) — unambiguous, Monday-anchored. |
|
|
# Weekly key uses ISO week notation (yyyy-Www) — unambiguous, Monday-anchored. |
|
|
# TTL set only on first INCR (Lua); daily=24h, weekly=7d — cleanup only, |
|
|
# TTL set once on hash creation (Lua TTL guard); resets are implicit in the |
|
|
# resets are implicit in the date/week embedded in the key name. |
|
|
# date/week embedded in the key name. Fields are IPs; no per-field TTL. |
|
|
|
|
|
# Memory: ~63 bytes/field vs ~148 bytes for per-IP STRING keys (~57% saving). |
|
|
# --------------------------------------------------------------------------- |
|
|
# --------------------------------------------------------------------------- |
|
|
QUOTA_IP_DAILY = 'quota:{ns}:daily:{date}:ip:{ip}' |
|
|
QUOTA_DAILY_HASH = 'quota:{ns}:daily:{date}' |
|
|
QUOTA_IP_WEEKLY = 'quota:{ns}:weekly:{week}:ip:{ip}' |
|
|
QUOTA_WEEKLY_HASH = 'quota:{ns}:weekly:{week}' |
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
|
# --------------------------------------------------------------------------- |
|
|
# Bot UA fragments |
|
|
# Bot UA fragments |
|
|
@ -115,6 +117,17 @@ _INCR_LUA = """ |
|
|
return n |
|
|
return n |
|
|
""" |
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
# Atomically increment a HASH field and set TTL on the hash key the first time |
|
|
|
|
|
# it is created (TTL < 0 covers both -1 = no expire and -2 = key just created). |
|
|
|
|
|
# KEYS[1] = hash key ARGV[1] = field (IP) ARGV[2] = TTL seconds |
|
|
|
|
|
_HINCRBY_LUA = """ |
|
|
|
|
|
local n = redis.call('HINCRBY', KEYS[1], ARGV[1], 1) |
|
|
|
|
|
if redis.call('TTL', KEYS[1]) < 0 then |
|
|
|
|
|
redis.call('EXPIRE', KEYS[1], ARGV[2]) |
|
|
|
|
|
end |
|
|
|
|
|
return n |
|
|
|
|
|
""" |
|
|
|
|
|
|
|
|
# Atomically write a block key and record it in the sharded ZSET index. |
|
|
# Atomically write a block key and record it in the sharded ZSET index. |
|
|
# Prunes expired entries from the target shard before inserting the new one, |
|
|
# Prunes expired entries from the target shard before inserting the new one, |
|
|
# keeping each shard bounded to only active blocks (amortised O(1) cleanup). |
|
|
# keeping each shard bounded to only active blocks (amortised O(1) cleanup). |
|
|
@ -289,6 +302,26 @@ def _incr_with_ttl(key, ttl): |
|
|
return count |
|
|
return count |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _hincrby_with_ttl(hash_key, field, ttl): |
|
|
|
|
|
""" |
|
|
|
|
|
Atomic HINCRBY + conditional EXPIRE via Redis Lua script (ratelimit cache, DB 1). |
|
|
|
|
|
|
|
|
|
|
|
Increments the counter for `field` inside `hash_key` and sets a TTL on the |
|
|
|
|
|
hash key if it does not already have one (i.e. on first creation). |
|
|
|
|
|
Falls back to a composite STRING key `hash_key:field` when Redis is unavailable. |
|
|
|
|
|
""" |
|
|
|
|
|
try: |
|
|
|
|
|
from django_redis import get_redis_connection |
|
|
|
|
|
client = get_redis_connection('ratelimit') |
|
|
|
|
|
return client.eval(_HINCRBY_LUA, 1, hash_key, field, ttl) |
|
|
|
|
|
except Exception: |
|
|
|
|
|
rl_cache = caches['ratelimit'] |
|
|
|
|
|
fallback_key = f'{hash_key}:{field}' |
|
|
|
|
|
count = (rl_cache.get(fallback_key) or 0) + 1 |
|
|
|
|
|
rl_cache.set(fallback_key, count, timeout=ttl) |
|
|
|
|
|
return count |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _set_block(block_key, index_key, ttl): |
|
|
def _set_block(block_key, index_key, ttl): |
|
|
""" |
|
|
""" |
|
|
Atomically set a block key (with TTL) and record it in a sharded ZSET index. |
|
|
Atomically set a block key (with TTL) and record it in a sharded ZSET index. |
|
|
@ -561,6 +594,11 @@ class RateLimitMiddleware: |
|
|
""" |
|
|
""" |
|
|
Increment daily and weekly API quota counters for all /api/ callers. |
|
|
Increment daily and weekly API quota counters for all /api/ callers. |
|
|
All callers are keyed by IP — auth status is not checked. |
|
|
All callers are keyed by IP — auth status is not checked. |
|
|
|
|
|
|
|
|
|
|
|
Keys are HASH structures: one hash per tenant per period, field = IP. |
|
|
|
|
|
This keeps the global keyspace at O(tenants) instead of O(unique IPs), |
|
|
|
|
|
reducing Redis memory ~57% vs. per-IP STRING keys at scale. |
|
|
|
|
|
|
|
|
Fails open (returns None) if Redis/cache is unavailable. |
|
|
Fails open (returns None) if Redis/cache is unavailable. |
|
|
""" |
|
|
""" |
|
|
today = date.today() |
|
|
today = date.today() |
|
|
@ -569,13 +607,13 @@ class RateLimitMiddleware: |
|
|
week_str = f'{iso[0]}-W{iso[1]:02d}' |
|
|
week_str = f'{iso[0]}-W{iso[1]:02d}' |
|
|
|
|
|
|
|
|
ip = get_client_ip(request) |
|
|
ip = get_client_ip(request) |
|
|
d_key = QUOTA_IP_DAILY.format(ns=_NAMESPACE, date=date_str, ip=ip) |
|
|
d_hash = QUOTA_DAILY_HASH.format(ns=_NAMESPACE, date=date_str) |
|
|
w_key = QUOTA_IP_WEEKLY.format(ns=_NAMESPACE, week=week_str, ip=ip) |
|
|
w_hash = QUOTA_WEEKLY_HASH.format(ns=_NAMESPACE, week=week_str) |
|
|
|
|
|
|
|
|
try: |
|
|
try: |
|
|
if _incr_with_ttl(d_key, 86400) > self.api_quota_daily: |
|
|
if _hincrby_with_ttl(d_hash, ip, 86400) > self.api_quota_daily: |
|
|
return 'daily' |
|
|
return 'daily' |
|
|
if _incr_with_ttl(w_key, 7 * 86400) > self.api_quota_weekly: |
|
|
if _hincrby_with_ttl(w_hash, ip, 7 * 86400) > self.api_quota_weekly: |
|
|
return 'weekly' |
|
|
return 'weekly' |
|
|
except Exception: |
|
|
except Exception: |
|
|
pass # fail open — quota not enforced when Redis unavailable |
|
|
pass # fail open — quota not enforced when Redis unavailable |
|
|
|
