Created
April 14, 2017 11:12
-
-
Save detorto/d60323c3c2fce911effa6cc7c2fa1223 to your computer and use it in GitHub Desktop.
Multi-process rate-limiter function. A mutex that unlocks N times per second is used.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import multiprocessing | |
#used to trigger the permission for request processing | |
RATED_LOCK = multiprocessing.Lock() | |
#used for messaging obout the state of operation: can do new, or can't | |
RATED_QUEUE = multiprocessing.Queue() | |
#rate-limit decorator, but works only in one thread | |
def RateLimited(maxPerSecond): | |
minInterval = 1.0 / float(maxPerSecond) | |
def decorate(func): | |
lastTimeCalled = [0.0] | |
def rateLimitedFunction(*args,**kargs): | |
elapsed = time.clock() - lastTimeCalled[0] | |
leftToWait = minInterval - elapsed | |
if leftToWait>0: | |
time.sleep(leftToWait) | |
ret = func(*args,**kargs) | |
lastTimeCalled[0] = time.clock() | |
return ret | |
return rateLimitedFunction | |
return decorate | |
def run_ratelimit_dispatcher(): | |
#yes, it is process. we need process not to interfear with main process. | |
p = multiprocessing.Process(target=rl_dispatch) | |
p.start() | |
#this method can be trigerred only N times per second. (per thread/process) | |
@RateLimited(2.5) | |
def set_event(): | |
try: | |
RATED_LOCK.release() | |
except ValueError: | |
pass | |
#to sincronyze set_event beetwin threads/processes we use event queue. | |
#when there is a message in queue, it means that PERFORM method was invoked and it locked the rated_lock. | |
#so we can try to unlock it, if rate_limit decorator allows. | |
def rl_dispatch(): | |
set_event() | |
while True: | |
RATED_QUEUE.get() | |
set_event(); | |
def perform(operation, args): | |
RATED_LOCK.acquire() | |
RATED_QUEUE.put("Can unlock") | |
print ("This will be printed 2.5 times per second") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment