http-request-with-retry.clj

(set-env! :dependencies '[[clj-http "2.2.0"]])
(require '[clj-http.client :as http])

(def ^:private ^:const
  retryable-http-status-codes
  #{408; Request Timeout
    409; Conflict
    410; Gone

    500; Internal Server Error
    ;; 501; Not Implemented
    502; Bad Gateway
    503; Service Unavailable
    504; Gateway Timeout
    ;; 505; HTTP Version Not Supported
    ;; 506; Variant Also Negotiates
    ;; 507; Insufficient Storage(WebDAV)
    509; Bandwidth Limit Exceeded
    ;; 510; Not Extended
    })

(defn http-get-with-retry [url & {:keys [max-retries retry-interval-sec]
                                  :or   {max-retries 100, retry-interval-sec 5}}]
  (loop [t max-retries]
    (let [[v retry?] (try
                       [(http/get url) false]
                       (catch clojure.lang.ExceptionInfo e
                         (let [s (-> e ex-data :object :status)]
                           (cond (zero? t) (throw e)

                                 (retryable-http-status-codes s)
                                 (do
                                   ;; (printf "%s:retry: t:%d\n" *ns* t) (flush)
                                   (Thread/sleep (* retry-interval-sec 1000))
                                   [nil true])

                                 (== s 404) [nil false]
                                 :else (throw e)))))]
      (if retry?
        (recur (dec t))
        v))))

;; (http-get-with-retry "http://example.com")


;;; or
(defn http-get [url & [{:keys [retries interval]
                        :as   req}] ]
  (let [retries  (or (:retries req) 5)
        interval (or (:interval req) 30000)]
   (->> #(http/get url req)
        (repeat (inc retries))
        (interpose #(Thread/sleep interval))
        (map (fn [p] (p)))
        (filter identity)
        first)))
;; (http-get "https://httpbin.org/status/500" {:retries 3, :debug   true})